diff --git a/clic/hits/v3.2.0/README.md b/clic/hits/v3.2.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d6bcd801d8169b5c2471daf49d06fbe703a35a0e --- /dev/null +++ b/clic/hits/v3.2.0/README.md @@ -0,0 +1,47 @@ +# CLIC hits: ParticleFlow v3.2.0 + +Attention and HEPTv2 checkpoints from the 28 October 2026 backbone campaign. +Both complete 50,000 optimizer steps with seed 12345 and global batch 128. +Training code: [`6b80b983477321cf708e5043fb1d2ec15dd398bc`](https://github.com/jpata/particleflow/commit/6b80b983477321cf708e5043fb1d2ec15dd398bc). +The v3.2.0 directory is a release label; this upload does not create a code tag. + +| Backbone | Original run | Parameters | Validation F1 | +| --- | --- | ---: | ---: | +| attention | [clic_hits_attention_seed12345_20261028_111202_121851](clic_hits_attention_seed12345_20261028_111202_121851/) | 7,260,244 | 0.7747 | +| heptv2 | [clic_hits_heptv2_seed12345_20261028_111202_012645](clic_hits_heptv2_seed12345_20261028_111202_012645/) | 12,786,964 | 0.8623 | + +## Files and checkpoint loading + +Each run preserves `checkpoints/checkpoint-50000.pth`, configs, the resolved +`particleflow_spec.yaml`, `model_kwargs.pkl`, scenario manifest, every training +log, complete available histories, separate `runs/train` and `runs/valid` +TensorBoard event streams, and `plots_step_50000`. The checkpoint contains +`model_state_dict`, optimizer state and `extra_state` with step 50000 and resume +state. Use the accompanying configuration with compatible ParticleFlow code. +The original full checkpoint bytes are retained. No best-checkpoint selection +or inference conversion is implied by the filename. + +[release.json](release.json) records dataset versions, model dimensions, +checkpoint hashes and metadata validation. [SHA256SUMS](SHA256SUMS) covers the +release contents except the checksum file itself. + +## Interpretation + +PF/clusters refers to the track/cluster-input MLPF model, not Pandora output. +Hit models reconstruct particle sets from raw hits. HEPTv2 and attention have +different parameter counts; hits and PF also differ in decoder, objectives and +target populations. All measurements are single-seed observations. + +Particle F1 uses capped matching cost and requires deltaR < 0.1 and relative +pT error < 0.5, without requiring PID agreement. PF uses TFDS 3.3.0 and hits +3.2.2. Validation uses 1530 PF entries or 1536 padded hit entries. Detailed +momentum uses a 512-entry subset with different selection across representations. +CLD hit HEPTv2 is missing its 30k history JSON; the final 50k result is present. + +[Comparison summaries](comparison_metrics/summary.json), +[metric definitions](comparison_metrics/provenance.json), and the archived +[study methodology](comparison_metrics/study_methodology.md) explain sampling +and timing caveats. Relative artifact links in that archived methodology refer +to the original local study; use the release links here for uploaded files. +Predictions, batch parquet dumps and dataset caches are not part of this release. +Training timing does not measure inference or upstream tracking/clustering cost. diff --git a/clic/hits/v3.2.0/SHA256SUMS b/clic/hits/v3.2.0/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..4032aca5cf26e694db8a1c401ab7639add4bcf41 --- /dev/null +++ b/clic/hits/v3.2.0/SHA256SUMS @@ -0,0 +1,215 @@ +99b660ae60f1b3d5acf2ef6679d604bda1d115c449744612271e5ff198368843 README.md +75f3976dcdcc42c0bc91a5dc418775e4d166ae08f1f19cb8e7ce2846e6302f57 clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-50000.pth +37533ffecf9983cb7c30826ba730b5290a87fbe99233eb182262c5de441cbd0f clic_hits_attention_seed12345_20261028_111202_121851/history/step_1000.json +fb49575adc71a5d62c805b8ebfdbdafb81e31b8b2e9b8de6e011a53c1295d87d clic_hits_attention_seed12345_20261028_111202_121851/history/step_10000.json +3da729a3e3de6c8cbebbd3ab81581744ff3f3d36f1088bd37c07dac47003b4c9 clic_hits_attention_seed12345_20261028_111202_121851/history/step_11000.json +18cab47f66f1553adb999358a10aa3f236140e6357d33decb69a3b80ea8e93ee clic_hits_attention_seed12345_20261028_111202_121851/history/step_12000.json +985b1e453fb7cc84f971e0175907d06da6702441930c6b5ebd7c1be565e72eb6 clic_hits_attention_seed12345_20261028_111202_121851/history/step_13000.json +09fdfb4dbbbd9d8373af1381c60b15c20662d9bd54382253c096c765e283e7d0 clic_hits_attention_seed12345_20261028_111202_121851/history/step_14000.json +736816928430784e870f1c23d97ed54509833199114c49f158c06e548e498534 clic_hits_attention_seed12345_20261028_111202_121851/history/step_15000.json +1abda39e8b745a0e9bf87bdd9acb204717d01c60e327e966bda12d05da5e7105 clic_hits_attention_seed12345_20261028_111202_121851/history/step_16000.json +fa5c6191e698cb7da2dbe591d31ca62afc8d64c61e45a43b8ad27591ac7bfe5d clic_hits_attention_seed12345_20261028_111202_121851/history/step_17000.json +28929d2cfcf6bc4bc0c50a458bf92f7aacfce35b2519442a95fa4bc3ca32c28e clic_hits_attention_seed12345_20261028_111202_121851/history/step_18000.json +885639f2c19d5c39f0767ea92ec69efca65bfb06084ac61c525aee12b940019d clic_hits_attention_seed12345_20261028_111202_121851/history/step_19000.json +fbf3c31ca0f8266a6ca25991177aff115475f227d7e8ad70da03de283d396057 clic_hits_attention_seed12345_20261028_111202_121851/history/step_2000.json +e0b3d2d3d05ef36a9718d1a1ed7f72934d4e84160be50e754fbe6197ade8aceb clic_hits_attention_seed12345_20261028_111202_121851/history/step_20000.json +a31210e84bb10dcd1be4f7b5501a20319478a671e516bee3dc13a1da97f3e314 clic_hits_attention_seed12345_20261028_111202_121851/history/step_21000.json +3c01c6a905e8f8bf5eb53d81215ad4cbd3909cab694739789eb193e04181f015 clic_hits_attention_seed12345_20261028_111202_121851/history/step_22000.json +dff212608376512f61e5117df128de7a16b6f4cadf1f274719a796095ef53d24 clic_hits_attention_seed12345_20261028_111202_121851/history/step_23000.json +38c0eda103b754d84ea1c1f0ceabe307971b5fa6e1fad9d5e87eff7ffafff635 clic_hits_attention_seed12345_20261028_111202_121851/history/step_24000.json +7e99e32374e1cc11ebc0c1340059f00ea00e7c4835bd85b359797b0301b9deb9 clic_hits_attention_seed12345_20261028_111202_121851/history/step_25000.json +d6ec5cb8d60d94f1173c968511687770350cfee81f18deb8df8d88f63349e4d4 clic_hits_attention_seed12345_20261028_111202_121851/history/step_26000.json +d38a2085680c140fd3b90be3f5aa6b539e1b1bdad3d72af4f9abd71c6c5e1e81 clic_hits_attention_seed12345_20261028_111202_121851/history/step_27000.json +1e50de2530d9b73368c0a9d2455ea69b65dd42190b43575d0af4e64891421472 clic_hits_attention_seed12345_20261028_111202_121851/history/step_28000.json +0ab274378c4f0996ae3df907ae34c4f9747b435452db74c02a76f0603750482b clic_hits_attention_seed12345_20261028_111202_121851/history/step_29000.json +3d571a07606798950596d4313329c33fdb6be46ff4628380ba557e6974924ec8 clic_hits_attention_seed12345_20261028_111202_121851/history/step_3000.json +3d9dbc38f2061d137f17aff7ad00e4bfdf80d502b2b8c891e8d59570c440924b clic_hits_attention_seed12345_20261028_111202_121851/history/step_30000.json +526db7479fbf66ba2c5da8a78f50a4a9ff172f4407eb9cfd78843880511e038e clic_hits_attention_seed12345_20261028_111202_121851/history/step_31000.json +319b3d1cadaca1e30bfdfc1800bfc8c4251f9bdc1543e0189e687accbabf9479 clic_hits_attention_seed12345_20261028_111202_121851/history/step_32000.json +c0daf974bff6b547fab33ecd6a26db4c6d4d606088c62b2ae9e33f998b5bed6e clic_hits_attention_seed12345_20261028_111202_121851/history/step_33000.json +074f558988371e4b680b533fc7825179568aa9083ebd96af18e1db4769735b14 clic_hits_attention_seed12345_20261028_111202_121851/history/step_34000.json +35e9551ad5fc8dd0448997074a45390ad2ae661461fa5eabd564f91aff042f55 clic_hits_attention_seed12345_20261028_111202_121851/history/step_35000.json +96eeb2a58d8f2b0fcb992981ee0ebd313cfd4bd8ebcc7c37ad814f7ee9a5c55f clic_hits_attention_seed12345_20261028_111202_121851/history/step_36000.json +0c67eebb50ac73a828c818157205f0cc98eb8fb82d78feb1f10ec0c838aed904 clic_hits_attention_seed12345_20261028_111202_121851/history/step_37000.json +94d7760309b5bca0aa6b5550d6bb4c10d3ef57110c546fc2f482bc5de8ab99af clic_hits_attention_seed12345_20261028_111202_121851/history/step_38000.json +29ee88d81183651a7a302f950eeb237c0f9e223766f5c093d11a8de08e8a53d5 clic_hits_attention_seed12345_20261028_111202_121851/history/step_39000.json +32b1d1759d3acca46140013c18c5b4d11042cfbe347793938592b17a5b1638ae clic_hits_attention_seed12345_20261028_111202_121851/history/step_4000.json +6c5a9bbbf0c54be0060c91489afdf124c214439d6e6af6d6c2c59bdadeff0650 clic_hits_attention_seed12345_20261028_111202_121851/history/step_40000.json +3302ba2cfb23d28f80653ae1b6b5cce306b1a8b259f1619d1cfa1e735789dce3 clic_hits_attention_seed12345_20261028_111202_121851/history/step_41000.json +4a6b31feaa713c6f6d5dc4ac56f42a7fb0924cd94a25d34c9222d007810b2a3f clic_hits_attention_seed12345_20261028_111202_121851/history/step_42000.json +4d35f31fd9681c0e1b24b4cbaa3c3b9f7287bcac4c80fd8e4c8f4b1b789aecbb clic_hits_attention_seed12345_20261028_111202_121851/history/step_43000.json +84c09bcc072ecd166dfe384558d84652b2d45ff03eda3fea03dc73a71b53ebf4 clic_hits_attention_seed12345_20261028_111202_121851/history/step_44000.json +e3c34a8a347e3c7dd07dedcb899ff414b314078a232dac4667b8e23fc40539c6 clic_hits_attention_seed12345_20261028_111202_121851/history/step_45000.json +c4f344f7a1fef1c8a0c00371205a8f7e076e9400cff3e44d4b7e44d36de4512b clic_hits_attention_seed12345_20261028_111202_121851/history/step_46000.json +bea2ec3ca550c46f97c951adc77190a4930e2dac38ddddf432a5a84bf888e380 clic_hits_attention_seed12345_20261028_111202_121851/history/step_47000.json +c691c0c2b6cf2913280cdbe925f115d99d9463df00a56ed59cf5307e884ba4ee clic_hits_attention_seed12345_20261028_111202_121851/history/step_48000.json +491ce7fdcfd22650aac8160005d86bf4dc809603336f8b3cedeb4bdcccf49653 clic_hits_attention_seed12345_20261028_111202_121851/history/step_49000.json +d6363afdf9c26dc968f814d9f725f3fb556c6e1573da233e40cb05beceb9ebfe clic_hits_attention_seed12345_20261028_111202_121851/history/step_5000.json +35543db47c3f5cc9ba7cb80f84868eeb6266dc945214a5b60abc4e9755e7306d clic_hits_attention_seed12345_20261028_111202_121851/history/step_50000.json +feb2afc320ee8d6461e59769486f32852355317191284bfda352bae2e8de6db5 clic_hits_attention_seed12345_20261028_111202_121851/history/step_6000.json +769e6e4bd3d5b9ec4f9c4404879f47bd90823a065bc86451781539e1f9f00dde clic_hits_attention_seed12345_20261028_111202_121851/history/step_7000.json +4ce35f0cf68e2d0cdc6ef731f0d47c373f6d225f5b4f4c9246999205697eb6df clic_hits_attention_seed12345_20261028_111202_121851/history/step_8000.json +b86c68bdc9f6f95c93129972f1aace04d0b64a96f112695ec83ec4568f025d2b clic_hits_attention_seed12345_20261028_111202_121851/history/step_9000.json +9e5513d7fa117a1c9fb90f1e1e0a10dda972838eeaa2c37cfadbb9bc14a9e610 clic_hits_attention_seed12345_20261028_111202_121851/hyperparameters.json +af27ac504a3687a3076f06e5d6db091587b102da9f607ff8111fbb6096bc2bd2 clic_hits_attention_seed12345_20261028_111202_121851/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 clic_hits_attention_seed12345_20261028_111202_121851/particleflow_spec.yaml +26dad0c47acab7b072961efb036434131d68e5cc976fab15046f85a5fc1c7d29 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf +e4c5411ac0f6afa5e3cbab9d721c724f67b0c0a27f1cab4f4a9d3eac50cc1d75 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.png +da2cbb435edb4e66f08da4b180b2eb55db9f986a2359b4b05ed93fe31b2c0833 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf +66ecef083177c37afd9f6f79cda3877ed476ea001bfcec4e113771e6ea1f94f2 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.png +491e4ae783ef06c61face60eb4a565bedf339ba81004b1f276947faeb8cff6d7 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf +137eaf4e2824c63d130cddf6428590feb16eb6ffcfb669dd117ef81f4eed2b8c clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png +5cfbe015bb95da3313a58d1c3bed62b5f33c14a158d772fd45ff1b95415551ff clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.pdf +a1a7bbdc7676a757df57d7592b25bb17bcd2a32e2732f7897d572e94f4903d37 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.png +fc7dee1f2ee54562f511018096b28eda15b836329204c02b7b2df6b8a376c829 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf +a3450ba6795b1622b4e76eda7fcf5f12f3e1cd4deed68365a8412e6fa6d1d1f9 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png +ff9ff37869e5943c823392909f3c998c6a9651d29cd307dcc23ee9fa624dd6c3 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.pdf +53e72c20375187f9362756e9c5f5adee1da6f7bb48c39cf5aa9423c5ceccff97 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.png +d35950fb78decc821ce14000e77786f3ee8899db2203cbbb805e920d13d0be32 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf +7cac37e0cd65dcb0c598e53ca67385977ef713a4f2afc20a24cc9ce4ccb7d2fc clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png +f499e9f1589b190caa5b30ce346dcba2f9e78443238564cb3d401b6bb9ebe33e clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf +031930784e9336f2f79fde12ebc3991b74f698c0060730167dbc63b9e7338a3f clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png +06ed589993f55dcea7ec0fcd1e26e835a1c9c9a0366ecf93fe63eb7471935a18 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf +347105deea795d3f5903b4e2fba5f5c694aff5310652593cbbc7eab1f226155b clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png +30f597955155cb8d3d4aa3a87a3b3b8440249263772c7633e8af6748813f15f3 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf +d222a74973b36126fe95f09259f283642bff8cca8a4dfa05e7cba86b04815efd clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.png +3f4efbae47760f95a76e6f058409ec61a96affc7cac2034518288790319df48a clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf +986dcdd574bd0fb56b4e5973264f83a3cb44d42d8b8a4b003b2d63b81ae88ecf clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png +f1e9e509b214f8b4d02d2cbf0b3b878a8f53a3373a6d54ec91f94357fcd51cd5 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf +0cb3f9afb5e2f137becaa0032d3189a7e1fe61abcb5f2d78d58ff5234f393095 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.png +18939f2503e219401e8c162688ab780fa378d1b68b63dd2aa9dcf07568f2e050 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf +40bc5c2656e48f5d9628e5df14c8b3023c7968fde4d4a8fa9e833e94b953d365 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png +8b554f65bf636f8b039bf813911962b55dfac473b669f74888c5f8c5be4410e4 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf +3c04363951ce8cf8bb46025fb7a4a8c48ff6aabfe96db4743e9f112f44a95d49 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png +31b16fdd4ba3d1d406f0f2a73ee91134fffa225bedeb16b614cf3aeee68b11ca clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf +b0ed465278dd736c655bef8f2bdf1cdcb0f5d5ef78ea097cc232d80c697ceb1c clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png +3728c2b3d2f84f78da49f4662043d1c6b0c2c47bbd4fce7a9fa7c139e3f5a8d1 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf +c9a7aff879935eaf80edfd9ce96a04c500acf391d4c3036ef06117d5d2b5edf5 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png +1d3801ad2b3cacee3e43ac766204c9ae23cf37a684695579499e42b7d13bb874 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf +318dacfae024a1489d11ad6bce6bd8a5687ccd444e6a8ed41ec374cf911b058d clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png +739a7a4f3a8c1dbaf1fc7dba81ab5b7604e19d8ea76725781421395310c9735a clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf +5676500176b0b005d97ec02091a15ef3d31dbe23ef8776f1de7f74ef6790afa6 clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png +00bd07dae8bf81ff30f0602ec17148c586e55643c640e9945697194ab3324ba6 clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793178800.nid006678.68011.0 +a9e03f7f9955d3d4c8b3c095eae4f49a90214fa347ce27436e195ba8ea346240 clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793278220.nid006214.91933.0 +b002c7c409904cfabf3998335adb693f84ef829ec524eb0e65a62ff47bcf975f clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793178800.nid006678.68011.1 +2ef8091d5387a73f97aab21dd5435748297078e6c12b0cf205b07a63bc6412f9 clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793278220.nid006214.91933.1 +6d05223dac45fef2fb11a7a550ec3f2c1269bbe1a4617538e327ba1010f17664 clic_hits_attention_seed12345_20261028_111202_121851/scenario-manifest.json +39063ac2356cdfee121ae115805e634f68ffb03747921ae608b635a0a73701aa clic_hits_attention_seed12345_20261028_111202_121851/train-config.yaml +2ad9f02fadc892043d48e3282ed2a07cf6e4fa11b49f4c607359f73dc89efa01 clic_hits_attention_seed12345_20261028_111202_121851/train.log +ab3827bfe699fd7fee333e430307b28c78154bdd8769652503232d8123e179c5 clic_hits_attention_seed12345_20261028_111202_121851/train.log.0 +e4094346763b7c99208b02e0fc8fdd83755ffc9e5278098b4c01b7c32b9892fc clic_hits_attention_seed12345_20261028_111202_121851/train.log.1 +7294c15d45473ee59c1a45852acd9b26f6f4a2683c9fdb12ad059259c41fbf01 clic_hits_attention_seed12345_20261028_111202_121851/train.log.2 +a5ae7c9ebf8dc0f75d6f5359a5bfd37899aa9cb893030336d32a3e9eb9819adf clic_hits_attention_seed12345_20261028_111202_121851/train.log.3 +733a635d6ad59433e51142bce734b59ad428709819db44fd4c64e9434d4a0505 clic_hits_attention_seed12345_20261028_111202_121851/train.log.4 +12d580802181c8d09e8b97b8f2bdffcc83c658f823cc4f74e084fcac09c0fac4 clic_hits_attention_seed12345_20261028_111202_121851/train.log.5 +16fbe08803850a9f6e4eb9e58dae4f1a969eeac50e1d3710e92248ce2e671de5 clic_hits_attention_seed12345_20261028_111202_121851/train.log.6 +6d5cddcd6ed6329f6a1fb9bc694e7cb9df6b8032ec33570a409cff9767c7185d clic_hits_attention_seed12345_20261028_111202_121851/train.log.7 +00392bfbe6a32174783b909254db2ed09959cb988650c35d48fd29d51e5dbf75 clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-50000.pth +caa805d9070868b98e9f6205351c4f4c4f044abf953f751d365c575ed8cb1a38 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_1000.json +0c5dd87a9f4bf61840f6a342aa18ac391646fe7e250afeb01b3604a9a079598b clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_10000.json +f4dc4610976cfa573ceb5c75d1b2f0d3329fcac57822a60fe2750e651e4df882 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_11000.json +e067a3fe14242a57fb157d854070270b0eec36cd8d86ff46b40055fd75885b27 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_12000.json +fd7fa8ea3410c685d33feda401ed4aa10ec5128e860a397c2e4b7d41f3dc84ea clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_13000.json +ddab9ffec22a57b396d9ac4a4001d6928f7f20d7fe55b7ebd80457e54e1d1405 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_14000.json +d0d98519db5d4da1a34fd76099c3842aab166282d823f9955a17f10f2aeaafc3 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_15000.json +928b117ce362b1e1e39d4651d055a7ca2bca56105ed7bf99dd3b8989be0ca4bd clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_16000.json +f92f2dfacf95aff079da5a8a2b69b1920d8ff6c153fd01efb71f439e6bcbbb48 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_17000.json +77a3e5b3c971dd6e2e723945bb00b4762d74919c80413eab26e7b4c4dff9fd18 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_18000.json +24e48dd179fefa194a5fb7cb2ff1abf1eeefd16ee74ce57c5437458a4fee5cf1 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_19000.json +d2cebb7a485f94eb9a268aae87ca9901b86aecbc9e574d78e1ae30b696737682 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_2000.json +8f7995e9ad0ffe93f8fd55b04f97b84bb66b121fc675f9971fe1537e40e4e079 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_20000.json +e05b86584367f4047c69443be2a970203160829f374130d19736f6ea8c96ce0f clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_21000.json +34f435d88505ff387f96ba66fed6cf49346db415191ddb11c60a870006c0b2b9 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_22000.json +16159a61a52541a5666cb6de1af39a9631d5e15873ff13b0bdca58fb76a9b613 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_23000.json +f4b3cf57d1b59c1390fc9693dbdfb627318bb725b8a670b4d0d9a27050c0317b clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_24000.json +9ae68d142fae506410973c7b1f0a8415a69aa2dea59420e133268c7ddd90736a clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_25000.json +3c0933c1157d9d667b0d6394a35ebcd2c84d9305e89a4299209016f4115293fa clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_26000.json +8b2b4af1208c44d383f6f14954b3988d17bf6707d4edf88b273a07ce2c1610f1 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_27000.json +5ecde1fbe69c372a99fd921f2efe57873e0f59678f1c52e49fe7551ae882af67 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_28000.json +64739fda89c9427243060d7be8e13a797f7957afff1a79be3a7f4474e28acd0e clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_29000.json +b2222a17733b25b3d49734f04dc64c2cd92ec0d85a040c4d71df054f4f14f72b clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_3000.json +0fcab505a1fec89d82dc2947957748aedeedcd73231b4aa9ceddf37c8330aa98 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_30000.json +48bedfa224af53082d2b23155ecfdf3134d11f9a46c13078063798d3bdc30d80 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_31000.json +da692fa62dc10cfeda6502fce681d11b11f9b32f58579637337628f390257df2 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_32000.json +3b0b608b7532e29158121cfd90141ebb232d94687b5497dff0e0a70798ec16ba clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_33000.json +c8b595ecc8023dfa7e6e2833d4e53f4b7ce8abe07d359aad2fdb5354a4f3e53d clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_34000.json +43e8727ea07d9015ad2dc72195b5520cf68ff8378943690509d9d80690d9fa23 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_35000.json +ec5c795c6acc8882b6c1062ce517e594f4e8ef1371934e6545e56f1fa84d01d2 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_36000.json +e373d6894a455ed63a233a4ccb8f69883c8efb645f15b4b690f79a06c210c3c2 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_37000.json +c547287db3be0564f4313d8f1c3dcd694c57400d3ed459837da4e87218e6f564 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_38000.json +3afec84d3ea82e49e5227f88bdc3ed4f9bf22f4879ead49dbe3b59fe13058452 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_39000.json +a781949a308cd342800a2092c67900d25c0fe0bc30e7808fe27d27dfc5fffdbc clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_4000.json +57886ed9539f60d5a88b5e30c463aad904a84bc4257367fd68d9244ec28dcb1c clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_40000.json +ece8dc3f004e6a496a88d6351f571e69262ca57bf87009b44a32adda04c8cbfa clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_41000.json +78f88fdaf38ea5b60723be360c9afdaafc5700940348872b53b6ef83baab24f9 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_42000.json +2c7f93011cc752f3951eaf1011afe7dd4c2354dfb166c072207fc89a7ee2bb40 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_43000.json +e57858618e4635f85d3849e508439890e4cb5e4880f0947b51b7b6d2ef696125 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_44000.json +d264480e92e9749f58aa3afed9647ff0749422c3abc9557ee7f0e1f7e06393ea clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_45000.json +df42e26e587502c45a67bb82b5821eede0023b3f31025bdbfda886fd7b67a477 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_46000.json +0298cea57055171a55483f2aca227a3eac73a2e5233ee0655a16c75e13f633ca clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_47000.json +ac5cbc5c9ddf90e5a653014f35c8f6f8d8a5ab996389d8f491cd6a22df3893c9 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_48000.json +e224e04f823e13f6a44d94161060c0778b402064e759b2ece1e6f926d88b1bcb clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_49000.json +a47235f0514ec52e47d28918107de432c5a3b3024733664916079a383541a127 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_5000.json +bae2ad29c3b73a5de2d5c8d8a9269fdde32d58814a64575786d197c536678cff clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_50000.json +9c5bcc6cc496ab74ca42604a9f8bb0d8eb524184454b38692a7e08e1909f629c clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_6000.json +8dcba2ba77a011ccb4dd591615ef2810d7e843c293e5c6d0954807006ac1de45 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_7000.json +e094fcd7a0d18a01a896be9be457ada66e02e8c350ef1d1b3301b1b524ff9004 clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_8000.json +c3509d32628307e230e225b6834334fbc726e3c903fd71aa5bc1a7f13ddcfbcd clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_9000.json +b5cdf3eaea5a633cdb739fb9ff018839fb826910b260e0796d19c15fc82d6f36 clic_hits_heptv2_seed12345_20261028_111202_012645/hyperparameters.json +88528787cbe2db8ee76475a6c6d3a94fb412233b068a26a15e912a35241265e1 clic_hits_heptv2_seed12345_20261028_111202_012645/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 clic_hits_heptv2_seed12345_20261028_111202_012645/particleflow_spec.yaml +3026706624f8869359be08687a4fa5a1311f32418f4b127708aa50290b3e522c clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf +8a3adb0b6069ece87e4c90c2f1d65a55ce77a27c06cb0de7cb46456780f28fbf clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.png +4d4fed7c9219439f1bc334f058349f04f4bc6e9a7e40440cd3b5f1f78654a696 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf +302437b9c8003c18c18771b16d021300fbb1e4c1539aafa2e43e7ea87448808a clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.png +0ed6badcec52d60d797de18fbb51f8c4df604dbc5cf3f93be65b615a9b5d5942 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf +c1bcad4575600f825ee46229e42ceaf5dbae238d7153ea3792bbc8fb356c13f5 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png +58f3a8ea7196c67e046f328e79be3118092a7d3645708023467ddc91a4e7b0f9 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.pdf +b3649d13a174be24678f3b1738a9c0a52f4e9e916fb7707718143949cbbe956d clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.png +9b423b77903737165c4a6a603df08e4222b7efbf40850111e044dadd6420145b clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf +f7d60c7df4516d4d2bf4f7cf00201dfef773fb283e2d23f631580087447491be clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png +08765d1a8b8641f4b1cc127d9bd1b3758263b7f4627a9b26685ff3e29a035ff8 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.pdf +53e72c20375187f9362756e9c5f5adee1da6f7bb48c39cf5aa9423c5ceccff97 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.png +f20e5ea61aa48c81d13bd84cd0b82837000219683f62809afa0a11aa39ab7d4e clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf +bd6f739be6297046ad324b6a0df9b52f57e2fadbe35109e0d1e5057cb6810bf3 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png +5c01624fdf65dcefffd04a4306591cc69dcb556f5aececd4f24be101ad671a49 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf +336df8a339d9765b580fead07ea12f00c209590b6aeb47f065ffd1ef8799c7ab clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png +a229070e3994b8d2b6c4b66a0870d0d99ca0954535b690d5d3c58a5fe125dcfe clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf +a993d678b54c9b008529581fbd5890fdb5969754bc8d5146b419558168805d2d clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png +3766a610213176b9df4fc8646244b024ace661f3d112e152dd8cfd430d7b6c67 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf +0b06f8713a57380f18dbd0a2df76fe6608b906b8a063c882534fb4b65d91dd3a clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.png +216b6af5a6d733cc410378d8b2bb1a5479ffe38dd186431e9e876f678b8e31b7 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf +18271f3a95cbcd375a052128c5a4a1980a793046e276eae8d0e1b347ce1d4d33 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png +419b0dd932e2bbdb53c5949c7af1b3b1a0fa5eaa9a92301bea29b4c05cf57ddd clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf +0cb3f9afb5e2f137becaa0032d3189a7e1fe61abcb5f2d78d58ff5234f393095 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.png +cfb1e20451c5b9a989277cbf94b566a71c594c72fdaac7cfeff42f1464cf3dfc clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf +8e7ab232ec665f8af248de3a74390520c0cc9ce45c9c47a8da74d80e73b683fe clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png +007af8475e35b87f9588c6daaed44de49a9e471819c165366066a5e69686fa6c clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf +f6753f1390699077c0d5c3bec5f37a1d244979b79c2a0a31311919ecacace1a6 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png +8df4632fe0521cc3b9cdeda092eed1967a94561af6f0c6c85d4cb1edc4120f6a clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf +2117412efcf30d4f55c99012ed9804ee1cd49794fa5f0a047f103e3a1ce9cfec clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png +da0c823594d9d7c23279f13ed14ea628d0aadc202d2a72f2e9bc08c7a7973e90 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf +6e68f9d713e35cf8ae0025d93e436f572c97bf2ef9fc4cd96062b81842577833 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png +3753295d86fe2f0dae20fc91e5058ebcc1cc38592dd448e1c598f274b0ab38c2 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf +415ff75b0d960e56b7fcb3135ff7f7eaaf8f493013d5feffebfc3b8765e1f6eb clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png +5f9f79664164897d80589330199378223e9a7818d1a90d035ba89a02877717ca clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf +5676500176b0b005d97ec02091a15ef3d31dbe23ef8776f1de7f74ef6790afa6 clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png +2e9c4ca47c92c205a6d6a6855d0be1cc676d4b2c72b6dcb06265b3c65ab7c035 clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793178800.nid006984.123108.0 +2331584585894d63fbcbd2feb4037613dd54028ae5929dd4f843f037ff1dd1a3 clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793278220.nid006205.64228.0 +1f4813742d3858077937ced34ecb8d60f44478e60a1b1846d3dd28be3d4f0fea clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793178800.nid006984.123108.1 +8027f914aaae8f6a63b2cda85bfaf3d2e10326c974e6ae23a3a36c10015ef244 clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793278220.nid006205.64228.1 +e1cf138f65896b7e623684a2707ef692b72ace2ebd7a02a4f5ca3c463e6de307 clic_hits_heptv2_seed12345_20261028_111202_012645/scenario-manifest.json +fa1dba1d0061663ce60cd02f4bc660188500ec14449aaea9e94a700661482a2d clic_hits_heptv2_seed12345_20261028_111202_012645/train-config.yaml +82400e277be35dd555b5fa814f8e22af1f6f48982753a6ffb285a02c78c96e30 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log +75021f2adc2a86d24343200800e791f963b226b091afde69dade7528b3f91a06 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.0 +9a1243d56c1efb5a8ceffa5f35f5782fb85314530fb52e6780245d62aedbce09 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.1 +0cad16bf54adb628b004380eddb22399173e5cfbe271809609786dc20aa66089 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.2 +d76e12632ddf20ee1f7e6f9e2988fe15de4249aeacb4f766d48bb93bc6bec87e clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.3 +81251502c1ba2d2097a6abcb188bc7c593be702cdf645765541bf23aad76ddcb clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.4 +0ba9b88b9f8cafba8e0b7f1c0918c81d5443d0173a04e97cb9a30eaef1635c40 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.5 +f9a3cad1848e9236840cb13d22799daecc8b9e73ea2b7e7eb5a6ba30387d11a6 clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.6 +852fb6ef4c231cb796d7c58686e63713ed1f9e8ba3b82c2b02d990c4bd8dc08f clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.7 +a9a813daf88031a86204a1100e12ecad2caf059954dd8256d7d98e46b2171105 comparison_metrics/provenance.json +4b00f93f03bf042116ccedcd743f4f78b0d26cc18501fca9bc294975f2e4beb1 comparison_metrics/study_methodology.md +8c0f8ac582b91f1d3eeee208268d2beed4192ee6b525b08feb5c8cb34145f73e comparison_metrics/summary.json +ea6efd19a6daa3ed2dc65cf07daa766dab1323ae14a74b46244f4239cda440d3 release.json diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-50000.pth b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..314c25734b1096ceded7579b30b7cdb7edabf77e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f3976dcdcc42c0bc91a5dc418775e4d166ae08f1f19cb8e7ce2846e6302f57 +size 87327154 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_1000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad1e562d2f23b8bd947d0b36a29c070994809ee7 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.04737540269092287, "train": {"Classification_binary": 0.4249165654182434, "Cardinality": 0.30488649010658264, "Classification": 0.3604743778705597, "Regression_pt": 0.5745282769203186, "Regression_eta": 0.006163963116705418, "Regression_sin_phi": 0.002949879737570882, "Regression_cos_phi": 0.002615902340039611, "Regression_energy": 0.5704050660133362, "HitGrouping": 0.33684247732162476, "Auxiliary": 0.6140534281730652, "Total": 3.0938758850097656, "Total_unweighted": 1.9420540314167738}, "train_group": {}, "valid": {"Classification_binary": 0.44798147678375244, "Cardinality": 0.350380539894104, "Classification": 0.38210006554921466, "Regression_pt": 0.5630092223485311, "Regression_eta": 0.006935425102710724, "Regression_sin_phi": 0.003303366402784983, "Regression_cos_phi": 0.0032913663114110627, "Regression_energy": 0.5556659698486328, "HitGrouping": 0.3353885014851888, "Auxiliary": 0.6290570100148519, "Total": 3.281132698059082, "Total_unweighted": 1.9622868923470378, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8071687817573547, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.06628754734992981, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.16397468745708466, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.06459047645330429, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.04011375457048416, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5071313977241516, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8363630771636963, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.05836489051580429, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.11962046474218369, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.05597800388932228, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.0342097133398056, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41172483563423157, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.762590765953064, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.06566288322210312, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.16764797270298004, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.06379792094230652, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.04825965315103531, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6104224920272827, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7835527658462524, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.0652092918753624, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.1621580570936203, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.06335154920816422, "metrics/hit_grouping/calorimeter/all/object_purity": 0.0443781353533268, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5593616962432861, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025374844670295715, "metrics/hit_grouping/calorimeter/background_recall": 0.6240862011909485, "metrics/hit_grouping/calorimeter/completeness": 0.17090575397014618, "metrics/hit_grouping/calorimeter/energy_completeness": 0.21002861857414246, "metrics/hit_grouping/calorimeter/energy_purity": 0.20757323503494263, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5723552107810974, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7841149568557739, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.049905963242053986, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.11292923241853714, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.047133054584264755, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.05015810206532478, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5507912635803223, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7823697328567505, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.06877315789461136, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.1740659922361374, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.06682689487934113, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.03934919461607933, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5644105076789856, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7835955619812012, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.093887560069561, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.25395020842552185, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.09405913203954697, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.037384454160928726, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5722411870956421, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7718837261199951, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.07155340164899826, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.20255371928215027, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.06886115670204163, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.04361056536436081, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5928993821144104, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8264722228050232, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.31257009506225586, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5311355590820312, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3151668310165405, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.16880682110786438, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4819466173648834, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7906056046485901, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.026735933497548103, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.07151466608047485, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.025205306708812714, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.029281048104166985, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5332127809524536, "metrics/hit_grouping/calorimeter/purity": 0.16831697523593903, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8860664963722229, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.019518373534083366, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.02041025646030903, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.003500743303447962, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04923076927661896, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8044643998146057, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.030767833814024925, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.05872292071580887, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.029843885451555252, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.02016090415418148, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4184720516204834, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7801649570465088, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.06816409528255463, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.1706453263759613, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.06619402021169662, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.046642281115055084, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5783381462097168, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6900657415390015, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.25343695282936096, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.13077552616596222, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.09870726615190506, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6894799470901489, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8312963843345642, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.218171626329422, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.15309399366378784, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.07697934657335281, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4015703797340393, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.592869758605957, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.24052925407886505, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.11204817146062851, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.09653016924858093, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7878239154815674, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6436703205108643, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.24272623658180237, "metrics/hit_grouping/tracker/all/object_completeness": 0.12142414599657059, "metrics/hit_grouping/tracker/all/object_purity": 0.09557183086872101, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7242112159729004, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0007960787625052035, "metrics/hit_grouping/tracker/background_recall": 0.0005330806015990674, "metrics/hit_grouping/tracker/completeness": 0.131756991147995, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5779971480369568, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.62441086769104, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.24574032425880432, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.13876628875732422, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.13618919253349304, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7483371496200562, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6499419212341309, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.23853036761283875, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.10423331707715988, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.06279581040143967, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7233961820602417, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6797599792480469, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.2404935210943222, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.10119226574897766, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.04023140296339989, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.671521008014679, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6788074970245361, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.2650071978569031, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.12509527802467346, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.07931136339902878, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6952731609344482, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7073713541030884, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.31890812516212463, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.1836719959974289, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.08862769603729248, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6451398134231567, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5866946578025818, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.20030437409877777, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.10569614171981812, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.11797600984573364, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.775813639163971, "metrics/hit_grouping/tracker/purity": 0.12507173418998718, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9316936731338501, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02215464785695076, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.013844586908817291, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.006343699060380459, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05343179777264595, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7599509954452515, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.08812856674194336, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.04643471911549568, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.02388957515358925, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.46656298637390137, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5912923812866211, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.2903016209602356, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.1445811539888382, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.11585860699415207, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8446962833404541, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 0.375, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.19315975904464722, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5666666626930237, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5378932952880859, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.59632408618927, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5300971269607544, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.3628025949001312, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.0007076910696923733, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.5559623837471008, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.699999988079071, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.390657424926758, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.10830231010913849, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": -0.022137504070997238, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.3027898669242859, "metrics/particle/class_1/pid_accuracy": 0.7163301110267639, "metrics/particle/class_2/efficiency": 0.3037917912006378, "metrics/particle/class_2/pid_accuracy": 0.02511078305542469, "metrics/particle/class_3/efficiency": 0.4123527705669403, "metrics/particle/class_3/pid_accuracy": 0.9026585221290588, "metrics/particle/class_4/efficiency": 0.33589988946914673, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.3199999928474426, "metrics/particle/class_5/pid_accuracy": 0.0, "metrics/particle/count/bias_mean": -9.285807609558105, "metrics/particle/count/mae": 18.55533790588379, "metrics/particle/count/prediction_mean": 60.77018356323242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.3040381669998169, "metrics/particle/event/energy_response_mean": 0.8468213081359863, "metrics/particle/event/met_abs_error": 19.502674102783203, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2711668312549591, "metrics/particle/event/scalar_pt_response_mean": 0.8462771773338318, "metrics/particle/event/vector_pt_closure": 0.14779438078403473, "metrics/particle/matched/delta_eta_abs_mean": 0.025062736123800278, "metrics/particle/matched/delta_phi_abs_mean": 0.02173888124525547, "metrics/particle/matched/delta_r_mean": 0.03701682761311531, "metrics/particle/matched/energy_relative_abs_mean": 0.21829642355442047, "metrics/particle/matched/pid_accuracy": 0.7519673705101013, "metrics/particle/matched/pt_relative_abs_mean": 0.20506584644317627, "metrics/particle/matching/duplicate_fraction": 0.08236289769411087, "metrics/particle/matching/efficiency": 0.35073322057724, "metrics/particle/matching/f1": 0.37562763690948486, "metrics/particle/matching/purity": 0.4043259918689728, "metrics/particle/matching/target_coverage_dr0p05": 0.2864803075790405, "metrics/particle/matching/target_coverage_dr0p10": 0.4244651794433594, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.22607475519180298, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.44454771280288696, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03653795272111893, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.21672101318836212, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.2067330926656723, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.021070752292871475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24945229291915894, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.02278893068432808, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.489983081817627, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.024083729833364487, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.1280033588409424, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 3686.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8896331191062927, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.04254412651062, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.150812149047852, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.9286928176879883, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.43319636583328247, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.28964614868164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04096625745296478, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.3914222717285156, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.3308354914188385, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.170550346374512, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8031.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.3370519280433655, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -9.919221878051758, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 25.7550048828125, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 10.411182403564453, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.996605634689331, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.5507354736328125, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 22.54791259765625, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.199731826782227, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 35.949337005615234, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0404934361577034, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.24482733011245728, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.22054636478424072, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.02071678265929222, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.26177558302879333, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.07611548900604248, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 3.1866700649261475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.4846545159816742, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.068551063537598, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 381.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.750623106956482, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.1869113445281982, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9.630158424377441, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9.452570915222168, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5951859951019287, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 46.68113708496094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10649161040782928, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.562685966491699, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.5956796407699585, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.875691413879395, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1371.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.12611718475818634, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -29.844837188720703, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 25.693817138671875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.7867465019226074, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3548676371574402, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11114992201328278, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 25.24474334716797, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.432541608810425, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 47.92455291748047, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.036887720227241516, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.21877644956111908, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.2049766480922699, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.031496014446020126, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24488046765327454, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002382938051596284, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4761756360530853, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.049670785665512085, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.206206202507019, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 8393.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7938974499702454, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.709028720855713, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.8341028690338135, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.5929174423217773, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.43908095359802246, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 18.519031524658203, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.018865935504436493, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.4149093627929688, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.12090131640434265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.824568510055542, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 18976.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.39151933789253235, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.360445976257324, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 26.70122718811035, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 21.561649322509766, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.6664588451385498, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.009470224380493, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 13.407280921936035, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.4608551263809204, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 19.90892219543457, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.036894503980875015, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.21896497905254364, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.2059723287820816, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.028082337230443954, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24677269160747528, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.010674157179892063, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8589680790901184, "metrics/particle/momentum/all/accepted/pz_bias": 0.05540231615304947, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1587209701538086, "metrics/particle/momentum/all/accepted/samples": 12460.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.8672121167182922, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.163875579833984, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.203751564025879, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.971034526824951, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.4449573755264282, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.241107940673828, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.029353724792599678, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.843338966369629, "metrics/particle/momentum/all/assigned/pz_bias": 0.20325039327144623, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.696685314178467, "metrics/particle/momentum/all/assigned/samples": 28378.0, "metrics/particle/momentum/all/efficiency": 0.35203707218170166, "metrics/particle/momentum/all/event/energy_bias": -36.40361404418945, "metrics/particle/momentum/all/event/p3_error_mean": 55.60172653198242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 24.09202766418457, "metrics/particle/momentum/all/event/px_bias": 2.308196783065796, "metrics/particle/momentum/all/event/py_bias": 4.449056148529053, "metrics/particle/momentum/all/event/pz_abs_mean": 43.69139099121094, "metrics/particle/momentum/all/event/pz_bias": 4.228045463562012, "metrics/particle/momentum/all/event/pz_residual_rms": 58.854347229003906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.05168686434626579, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.24840280413627625, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22864840924739838, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.049386829137802124, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.26758813858032227, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.016839632764458656, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.195256233215332, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.11229608207941055, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.6098811626434326, "metrics/particle/momentum/class_1/accepted/samples": 5107.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.899084210395813, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 3.6024982929229736, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 3.944741725921631, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 3.6612050533294678, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.4510399401187897, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 16.51352310180664, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.039927128702402115, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 2.221877098083496, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.3010430634021759, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.820517063140869, "metrics/particle/momentum/class_1/assigned/samples": 13174.0, "metrics/particle/momentum/class_1/efficiency": 0.30604663491249084, "metrics/particle/momentum/class_1/event/energy_bias": -6.446139812469482, "metrics/particle/momentum/class_1/event/p3_error_mean": 44.075557708740234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 24.828784942626953, "metrics/particle/momentum/class_1/event/px_bias": 4.509156227111816, "metrics/particle/momentum/class_1/event/py_bias": 3.100026845932007, "metrics/particle/momentum/class_1/event/pz_abs_mean": 31.437713623046875, "metrics/particle/momentum/class_1/event/pz_bias": 7.102722644805908, "metrics/particle/momentum/class_1/event/pz_residual_rms": 41.741451263427734, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04211200401186943, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24723945558071136, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.24866628646850586, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.11153251677751541, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.28719091415405273, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020385051146149635, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4589476585388184, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08251350373029709, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 2.9834539890289307, "metrics/particle/momentum/class_2/accepted/samples": 883.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8493026494979858, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7260216474533081, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.011845350265503, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5514345765113831, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3914141356945038, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3344035148620605, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04848484694957733, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.66180157661438, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.034641534090042114, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.852194309234619, "metrics/particle/momentum/class_2/assigned/samples": 1980.0, "metrics/particle/momentum/class_2/efficiency": 0.2966073155403137, "metrics/particle/momentum/class_2/event/energy_bias": -45.42292404174805, "metrics/particle/momentum/class_2/event/p3_error_mean": 32.4720458984375, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.573410034179688, "metrics/particle/momentum/class_2/event/px_bias": -0.08020990341901779, "metrics/particle/momentum/class_2/event/py_bias": -0.5369537472724915, "metrics/particle/momentum/class_2/event/pz_abs_mean": 24.458879470825195, "metrics/particle/momentum/class_2/event/pz_bias": -0.13130153715610504, "metrics/particle/momentum/class_2/event/pz_residual_rms": 38.13360595703125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023829882964491844, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18921752274036407, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1796407401561737, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.004317486193031073, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21960322558879852, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022271715570241213, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.43964502215385437, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.006541442591696978, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2246270179748535, "metrics/particle/momentum/class_3/accepted/samples": 6286.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8393195271492004, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.221564769744873, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.952361106872559, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.813066482543945, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.44715699553489685, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.864492416381836, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013337492942810059, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2724753618240356, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11592404544353485, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.207606315612793, "metrics/particle/momentum/class_3/assigned/samples": 12821.0, "metrics/particle/momentum/class_3/efficiency": 0.4133622646331787, "metrics/particle/momentum/class_3/event/energy_bias": 29.764911651611328, "metrics/particle/momentum/class_3/event/p3_error_mean": 33.42888259887695, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 18.17188262939453, "metrics/particle/momentum/class_3/event/px_bias": -1.1949825286865234, "metrics/particle/momentum/class_3/event/py_bias": 0.282089501619339, "metrics/particle/momentum/class_3/event/pz_abs_mean": 23.965742111206055, "metrics/particle/momentum/class_3/event/pz_bias": -2.4397642612457275, "metrics/particle/momentum/class_3/event/pz_residual_rms": 38.930442810058594, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.048515286296606064, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.29750990867614746, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.27349838614463806, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.14445917308330536, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.30591341853141785, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.10400000214576721, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 3.3129374980926514, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030796324834227562, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 6.4535603523254395, "metrics/particle/momentum/class_4/accepted/samples": 125.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.905690610408783, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.318572521209717, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.760778427124023, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.402949333190918, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.46000000834465027, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.787935256958008, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.08666666597127914, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 3.3543429374694824, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.43774938583374023, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 7.548672676086426, "metrics/particle/momentum/class_4/assigned/samples": 300.0, "metrics/particle/momentum/class_4/efficiency": 0.3434065878391266, "metrics/particle/momentum/class_4/event/energy_bias": -8.169275283813477, "metrics/particle/momentum/class_4/event/p3_error_mean": 7.000441074371338, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 5.293708324432373, "metrics/particle/momentum/class_4/event/px_bias": -0.07486768066883087, "metrics/particle/momentum/class_4/event/py_bias": 1.1190836429595947, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.645153284072876, "metrics/particle/momentum/class_4/event/pz_bias": 0.015205562114715576, "metrics/particle/momentum/class_4/event/pz_residual_rms": 11.057531356811523, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.045707885175943375, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.25064200162887573, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.26655372977256775, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.14044909179210663, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.30316275358200073, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.033898305147886276, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2473032474517822, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.017147274687886238, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.648773670196533, "metrics/particle/momentum/class_5/accepted/samples": 59.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.49482861161231995, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.2074445486068726, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9721041321754456, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.35191610455513, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3786407709121704, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.0797719955444336, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.13592232763767242, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.351365089416504, "metrics/particle/momentum/class_5/assigned/pz_bias": 1.123507022857666, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.795661926269531, "metrics/particle/momentum/class_5/assigned/samples": 103.0, "metrics/particle/momentum/class_5/efficiency": 0.37106919288635254, "metrics/particle/momentum/class_5/event/energy_bias": -6.130187034606934, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.494208335876465, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.3324127197265625, "metrics/particle/momentum/class_5/event/px_bias": -0.8508989810943604, "metrics/particle/momentum/class_5/event/py_bias": 0.48480936884880066, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.7470719814300537, "metrics/particle/momentum/class_5/event/pz_bias": -0.3188173472881317, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.689043998718262, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7998114228248596, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.801791250705719, "metrics/set_slot/angular_recall_all_dr0p10": 0.7915450930595398, "metrics/set_slot/angular_recall_selected_dr0p10": 0.47392338514328003, "metrics/set_slot/soft_count_bias_mean": 4.759194374084473}, "step_train_time": 4.1507978439331055, "step_valid_time": 2253.464139699936, "step_total_time": 2257.6149382591248, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.047630107941317, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.511405661965735, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3323, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2124, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1732, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7016848364717542, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.639181462533855, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6689763779527559, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.36081853746614506, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.5721836802114305, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.5212157688835389, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.545511811023622, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7016848364717542, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.879412087889674, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4323435405578776, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2193, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1500, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1292, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6631299734748011, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6839945280437757, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6734006734006734, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.31600547195622436, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.5711759504862953, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.5891472868217055, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.5800224466891134, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6631299734748011, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7394462777457207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4213033244826635, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 766, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 569, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.59472049689441, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.6808888888888889, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.6348943224202238, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.3191111111111111, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.4417701863354037, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.5057777777777778, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.4716121011189391, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.59472049689441} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_10000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a95d9b183e2d25307f75389451b409140071cec --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47375402690922874, "train": {"Classification_binary": 0.3408077657222748, "Cardinality": 0.2411758005619049, "Classification": 0.17899024486541748, "Regression_pt": 0.29682329297065735, "Regression_eta": 0.0035721762105822563, "Regression_sin_phi": 0.0020714402198791504, "Regression_cos_phi": 0.001986285438761115, "Regression_energy": 0.2918890416622162, "HitGrouping": 0.24117156863212585, "Auxiliary": 0.3952028453350067, "Total": 2.0466511249542236, "Total_unweighted": 1.1161402470897883}, "train_group": {}, "valid": {"Classification_binary": 0.2879638870557149, "Cardinality": 0.19658331076304117, "Classification": 0.161402036746343, "Regression_pt": 0.29014265537261963, "Regression_eta": 0.0031706088533004126, "Regression_sin_phi": 0.0017407375077406566, "Regression_cos_phi": 0.001772292113552491, "Regression_energy": 0.28610273202260333, "HitGrouping": 0.241684357325236, "Auxiliary": 0.36262206236521405, "Total": 1.8452011744181316, "Total_unweighted": 1.0322949496718745, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7523046135902405, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.30236881971359253, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.421082466840744, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.29460713267326355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2777342200279236, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5874185562133789, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6984323263168335, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.27070727944374084, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3951202929019928, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2624191641807556, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2673843204975128, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6628261804580688, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7661052346229553, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.29412955045700073, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.39345109462738037, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2872418165206909, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2771894335746765, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5733911991119385, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7555942535400391, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.29468002915382385, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4024924039840698, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2873854339122772, "metrics/hit_grouping/calorimeter/all/object_purity": 0.27648526430130005, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5859270691871643, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025292886421084404, "metrics/hit_grouping/calorimeter/background_recall": 0.6243885159492493, "metrics/hit_grouping/calorimeter/completeness": 0.36607226729393005, "metrics/hit_grouping/calorimeter/energy_completeness": 0.40358173847198486, "metrics/hit_grouping/calorimeter/energy_purity": 0.39881524443626404, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5636977553367615, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7747064232826233, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3024439513683319, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3415347933769226, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2973216474056244, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.28934282064437866, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5424903035163879, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7431619167327881, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.31742608547210693, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4492964446544647, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3076040744781494, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.29997313022613525, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6120815277099609, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7280957102775574, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.25454288721084595, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.48274555802345276, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.24528400599956512, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2247934639453888, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6506130695343018, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7340053915977478, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.323453813791275, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5034568905830383, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3171677887439728, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.30043545365333557, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6329492330551147, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7607383728027344, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4488029479980469, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6206174492835999, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4346153736114502, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3973294794559479, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5672422647476196, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7781453728675842, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2440243363380432, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2660372257232666, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2365274727344513, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.23527374863624573, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.53776615858078, "metrics/hit_grouping/calorimeter/purity": 0.3604969382286072, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8945756554603577, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04236200079321861, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14923076331615448, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.042461540549993515, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.013832153752446175, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04153846204280853, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.816429615020752, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.07374598830938339, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09635119885206223, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.07111901789903641, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.05584516376256943, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3728620409965515, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7491329908370972, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3127651810646057, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.42549842596054077, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3050445318222046, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.29477381706237793, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6097849011421204, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7351655960083008, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4390316903591156, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3934015929698944, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.34769144654273987, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5947399735450745, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7878924012184143, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.38867077231407166, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45689699053764343, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.39544013142585754, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4408300518989563, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6743732690811157, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4350633919239044, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.34420543909072876, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.31514453887939453, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6819721460342407, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.703148365020752, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43241071701049805, "metrics/hit_grouping/tracker/all/object_completeness": 0.36924561858177185, "metrics/hit_grouping/tracker/all/object_purity": 0.3321939706802368, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6340922117233276, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004733771085739136, "metrics/hit_grouping/tracker/background_recall": 0.021915536373853683, "metrics/hit_grouping/tracker/completeness": 0.39106807112693787, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.588085949420929, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7103773355484009, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43189066648483276, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.42383453249931335, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.38472479581832886, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6294305324554443, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6955106854438782, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45155149698257446, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.35068780183792114, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3149777054786682, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6462973356246948, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.695216178894043, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4132281541824341, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.26778340339660645, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.23387503623962402, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6314724683761597, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7304272651672363, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4718727469444275, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3991033136844635, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3463745415210724, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5931883454322815, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7486612796783447, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.43608522415161133, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3864082098007202, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3584391474723816, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5619174242019653, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6596082448959351, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3803558945655823, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.32733094692230225, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3091055154800415, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7000702619552612, "metrics/hit_grouping/tracker/purity": 0.3726942539215088, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8870314955711365, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.010860121808946133, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.01789010316133499, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.018777044489979744, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07558644562959671, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6837571263313293, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1596682220697403, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.14214271306991577, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.13133907318115234, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5775012969970703, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6803696751594543, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.520748496055603, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.44284820556640625, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.39765259623527527, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7164276242256165, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.43257877230644226, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.49793708324432373, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5412663817405701, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5733534693717957, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.053148578852415085, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04254871606826782, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.429275512695312, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2558380663394928, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.12539827823638916, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.715195894241333, "metrics/particle/class_1/pid_accuracy": 0.9306977391242981, "metrics/particle/class_2/efficiency": 0.34025129675865173, "metrics/particle/class_2/pid_accuracy": 0.2980547249317169, "metrics/particle/class_3/efficiency": 0.5277926921844482, "metrics/particle/class_3/pid_accuracy": 0.9236647486686707, "metrics/particle/class_4/efficiency": 0.6140519976615906, "metrics/particle/class_4/pid_accuracy": 0.18338558077812195, "metrics/particle/class_5/efficiency": 0.7360000014305115, "metrics/particle/class_5/pid_accuracy": 0.2771739065647125, "metrics/particle/count/bias_mean": -8.749348640441895, "metrics/particle/count/mae": 9.581379890441895, "metrics/particle/count/prediction_mean": 61.306640625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.11640426516532898, "metrics/particle/event/energy_response_mean": 0.9825646281242371, "metrics/particle/event/met_abs_error": 11.285630226135254, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09450054168701172, "metrics/particle/event/scalar_pt_response_mean": 1.001604437828064, "metrics/particle/event/vector_pt_closure": 0.0888921245932579, "metrics/particle/matched/delta_eta_abs_mean": 0.018760323524475098, "metrics/particle/matched/delta_phi_abs_mean": 0.01893736980855465, "metrics/particle/matched/delta_r_mean": 0.029806755483150482, "metrics/particle/matched/energy_relative_abs_mean": 0.1858842670917511, "metrics/particle/matched/pid_accuracy": 0.8873385190963745, "metrics/particle/matched/pt_relative_abs_mean": 0.18432815372943878, "metrics/particle/matching/duplicate_fraction": 0.107001394033432, "metrics/particle/matching/efficiency": 0.6020761132240295, "metrics/particle/matching/f1": 0.6421771049499512, "metrics/particle/matching/purity": 0.6880010962486267, "metrics/particle/matching/target_coverage_dr0p05": 0.5443934202194214, "metrics/particle/matching/target_coverage_dr0p10": 0.675380527973175, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3797650635242462, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6524357199668884, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02843468263745308, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18236210942268372, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.18320924043655396, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02240179292857647, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2233762890100479, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013916500844061375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1205947399139404, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.10085144639015198, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7131311893463135, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6539.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3745384216308594, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.2136330604553223, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.8562443256378174, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.651498317718506, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.217473566532135, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 45.339820861816406, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02871452458202839, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.6872180700302124, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.11294242739677429, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.005929470062256, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8985.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5979334115982056, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.27657878398895264, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.562811851501465, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.8149189949035645, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.0389498472213745, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.12265637516975403, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.637361526489258, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.2390888929367065, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.929162979125977, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028020355850458145, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1851637065410614, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18297246098518372, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.012392677366733551, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22475308179855347, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.033986929804086685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.862856149673462, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07057824730873108, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.611114978790283, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1530.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.43721410632133484, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.290861964225769, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.7735984325408936, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.5689733028411865, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21680347621440887, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.0345458984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.060357313603162766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.890533924102783, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.36170846223831177, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.907562255859375, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2071.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5064548254013062, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.636340141296387, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.88034439086914, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.967238426208496, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.23043373227119446, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.18483957648277283, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.642683029174805, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7690955400466919, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.91954803466797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.030233334749937057, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18584567308425903, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1839524358510971, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.01041670236736536, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.22429978847503662, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0016565017867833376, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.39516907930374146, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02579066902399063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0951619148254395, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13281.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4228554666042328, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.98801589012146, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.530139923095703, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.3431057929992676, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2622675895690918, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.062105178833008, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.010007111355662346, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8457810282707214, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.017743462696671486, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.8230555057525635, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19686.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6195363402366638, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 8.117112159729004, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.632579803466797, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.4136323928833, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -4.050571441650391, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.0259808748960495, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.528537750244141, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.648198664188385, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.044910430908203, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.029523862525820732, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18472987413406372, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18365459144115448, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.014229055494070053, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.224049910902977, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007728337310254574, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.722528338432312, "metrics/particle/momentum/all/accepted/pz_bias": 0.0519896075129509, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1270480155944824, "metrics/particle/momentum/all/accepted/samples": 21350.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4097011089324951, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.6473538875579834, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.709218740463257, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.5158228874206543, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24611280858516693, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 29.338666915893555, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01886669732630253, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.229457974433899, "metrics/particle/momentum/all/assigned/pz_bias": 0.06873930990695953, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.8908722400665283, "metrics/particle/momentum/all/assigned/samples": 30742.0, "metrics/particle/momentum/all/efficiency": 0.6032095551490784, "metrics/particle/momentum/all/event/energy_bias": -3.2426488399505615, "metrics/particle/momentum/all/event/p3_error_mean": 30.550493240356445, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.28807258605957, "metrics/particle/momentum/all/event/px_bias": -5.319954872131348, "metrics/particle/momentum/all/event/py_bias": -0.036202333867549896, "metrics/particle/momentum/all/event/pz_abs_mean": 22.61606216430664, "metrics/particle/momentum/all/event/pz_bias": 1.118192195892334, "metrics/particle/momentum/all/event/pz_residual_rms": 35.70573043823242, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03387296944856644, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.19124573469161987, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1927913874387741, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.02688697539269924, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.23177878558635712, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00749937491491437, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7695715427398682, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.08937335759401321, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.042923927307129, "metrics/particle/momentum/class_1/accepted/samples": 12001.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2332114279270172, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0119454860687256, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.1670500040054321, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9534717202186584, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.17513541877269745, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.473779678344727, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014315192587673664, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.07206130027771, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.08445465564727783, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.330096960067749, "metrics/particle/momentum/class_1/assigned/samples": 15508.0, "metrics/particle/momentum/class_1/efficiency": 0.7191826105117798, "metrics/particle/momentum/class_1/event/energy_bias": 18.295574188232422, "metrics/particle/momentum/class_1/event/p3_error_mean": 26.30994987487793, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.761669158935547, "metrics/particle/momentum/class_1/event/px_bias": -6.526473522186279, "metrics/particle/momentum/class_1/event/py_bias": -4.149626731872559, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.910795211791992, "metrics/particle/momentum/class_1/event/pz_bias": 1.9628846645355225, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.15167236328125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04166816547513008, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2156963348388672, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2130960077047348, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06607723981142044, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2548879384994507, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04056795313954353, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.1523547172546387, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.021852899342775345, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.128558158874512, "metrics/particle/momentum/class_2/accepted/samples": 986.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.668891429901123, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9740356802940369, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.402137279510498, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.038448691368103, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3231511116027832, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 13.689286231994629, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05841371789574623, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9427714347839355, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.342621386051178, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.184199810028076, "metrics/particle/momentum/class_2/assigned/samples": 1866.0, "metrics/particle/momentum/class_2/efficiency": 0.33120590448379517, "metrics/particle/momentum/class_2/event/energy_bias": -25.486658096313477, "metrics/particle/momentum/class_2/event/p3_error_mean": 24.96347427368164, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.573721885681152, "metrics/particle/momentum/class_2/event/px_bias": 0.6316900253295898, "metrics/particle/momentum/class_2/event/py_bias": 1.4573651552200317, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.42110252380371, "metrics/particle/momentum/class_2/event/pz_bias": -1.0906150341033936, "metrics/particle/momentum/class_2/event/pz_residual_rms": 28.584009170532227, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02146448940038681, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17080986499786377, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1657620072364807, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01186536718159914, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20692984759807587, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0027424581348896027, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.43241649866104126, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01051347702741623, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3916409015655518, "metrics/particle/momentum/class_3/accepted/samples": 8022.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5890260934829712, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.861277103424072, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.7663421630859375, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.622970104217529, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3206976652145386, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 43.08278274536133, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.017209302634000778, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1370078325271606, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003802119055762887, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.789416790008545, "metrics/particle/momentum/class_3/assigned/samples": 12900.0, "metrics/particle/momentum/class_3/efficiency": 0.5275202393531799, "metrics/particle/momentum/class_3/event/energy_bias": 9.01876163482666, "metrics/particle/momentum/class_3/event/p3_error_mean": 17.353931427001953, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.836478233337402, "metrics/particle/momentum/class_3/event/px_bias": 1.2898337841033936, "metrics/particle/momentum/class_3/event/py_bias": 2.361799478530884, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.213248252868652, "metrics/particle/momentum/class_3/event/pz_bias": -0.32433634996414185, "metrics/particle/momentum/class_3/event/pz_residual_rms": 22.508575439453125, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03304941952228546, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18341004848480225, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18840743601322174, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.02767411433160305, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2266857922077179, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.4510748386383057, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4988749325275421, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.3941898345947266, "metrics/particle/momentum/class_4/accepted/samples": 228.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3641436994075775, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.465806722640991, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.2262184619903564, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.034306049346924, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24922119081020355, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.817371368408203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.034267913550138474, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6750434637069702, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5111024975776672, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.556463718414307, "metrics/particle/momentum/class_4/assigned/samples": 321.0, "metrics/particle/momentum/class_4/efficiency": 0.6263736486434937, "metrics/particle/momentum/class_4/event/energy_bias": -3.187934398651123, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.056093692779541, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.0123047828674316, "metrics/particle/momentum/class_4/event/px_bias": -0.25452136993408203, "metrics/particle/momentum/class_4/event/py_bias": 0.43523892760276794, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1518988609313965, "metrics/particle/momentum/class_4/event/pz_bias": 0.6299822926521301, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.779417991638184, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.026696588844060898, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.21337907016277313, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21702271699905396, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.042848359793424606, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.26185476779937744, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06194690242409706, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.3755743503570557, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.11984862387180328, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.619510173797607, "metrics/particle/momentum/class_5/accepted/samples": 113.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10142971575260162, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.34797966480255127, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3427385687828064, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04710722342133522, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.20408163964748383, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5065035820007324, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10884353518486023, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.2256035804748535, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.33320915699005127, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.9529948234558105, "metrics/particle/momentum/class_5/assigned/samples": 147.0, "metrics/particle/momentum/class_5/efficiency": 0.7106918096542358, "metrics/particle/momentum/class_5/event/energy_bias": -1.882393717765808, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.5631155967712402, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.7388529777526855, "metrics/particle/momentum/class_5/event/px_bias": -0.460484117269516, "metrics/particle/momentum/class_5/event/py_bias": -0.14097927510738373, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.8349164724349976, "metrics/particle/momentum/class_5/event/pz_bias": -0.05972355604171753, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.621547698974609, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5647230744361877, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5667533278465271, "metrics/set_slot/angular_recall_all_dr0p10": 0.8688455820083618, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6885024905204773, "metrics/set_slot/soft_count_bias_mean": -0.9404086470603943}, "step_train_time": 4.88034725189209, "step_valid_time": 25145.583852767944, "step_total_time": 25150.464200496674, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0505657118136629, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2480069674189932, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2990, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2739, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2626, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9048562933597621, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9160535117056856, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9104204753199269, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08394648829431438, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8675256029071688, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8782608695652174, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8728602293501745, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9048562933597621, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006056716145476, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22277785800834005, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2213, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2021, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2000, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8934571175950486, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9132399457749661, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9032402234636872, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0867600542250339, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8841732979664014, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.903750564844103, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8938547486033519, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8934571175950486, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00174519536281, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22676409477998027, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1119, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1094, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8687888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8987951807228916, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8835373075404659, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10120481927710843, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8493788819875776, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.878714859437751, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8637978681405448, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8687888198757764} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_11000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..3296e2726dfc2d339afd7a1617886a254d757514 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5211294296001516, "train": {"Classification_binary": 0.32866066694259644, "Cardinality": 0.24240849912166595, "Classification": 0.16474144160747528, "Regression_pt": 0.26201504468917847, "Regression_eta": 0.0030440932605415583, "Regression_sin_phi": 0.0015996798174455762, "Regression_cos_phi": 0.0017150116618722677, "Regression_energy": 0.26140499114990234, "HitGrouping": 0.23590876162052155, "Auxiliary": 0.38493871688842773, "Total": 1.9047242403030396, "Total_unweighted": 1.023180929129012}, "train_group": {}, "valid": {"Classification_binary": 0.27420000235239667, "Cardinality": 0.18037400643030801, "Classification": 0.14625796675682068, "Regression_pt": 0.25615207354227704, "Regression_eta": 0.0028464983527859053, "Regression_sin_phi": 0.0015522769341866176, "Regression_cos_phi": 0.0015420842294891675, "Regression_energy": 0.2560807267824809, "HitGrouping": 0.23590354124704996, "Auxiliary": 0.3429720401763916, "Total": 1.706166426340739, "Total_unweighted": 0.938631628950437, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7587807774543762, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3319982886314392, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43515631556510925, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.32562416791915894, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3077985942363739, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5671105980873108, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7137035727500916, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.30171579122543335, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.401897668838501, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2935698330402374, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2954908311367035, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6201287508010864, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7707465887069702, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3219683766365051, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.407086044549942, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3151000440120697, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3060894310474396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.547309160232544, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7617795467376709, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.32337939739227295, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4156534969806671, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3165554702281952, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3056887984275818, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5602127909660339, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025471659377217293, "metrics/hit_grouping/calorimeter/background_recall": 0.6251596212387085, "metrics/hit_grouping/calorimeter/completeness": 0.3957655131816864, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4284772276878357, "metrics/hit_grouping/calorimeter/energy_purity": 0.4234987795352936, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5427442193031311, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.784270703792572, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3313770294189453, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34589430689811707, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3270215392112732, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.31343623995780945, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5102418661117554, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7501779794692993, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.34844374656677246, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4600921869277954, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3382418155670166, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3350369334220886, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5864871144294739, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7262630462646484, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2803315222263336, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5170016288757324, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.271803081035614, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2587064504623413, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.638604462146759, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7384233474731445, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.35721129179000854, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5188415050506592, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3505178391933441, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.33947792649269104, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6092182993888855, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7497765421867371, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4679560363292694, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6582940816879272, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.45431092381477356, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.43966078758239746, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.569858729839325, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7884249091148376, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2685074210166931, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.27367082238197327, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.26241612434387207, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2522198557853699, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5062943696975708, "metrics/hit_grouping/calorimeter/purity": 0.38980942964553833, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8837612867355347, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0503847673535347, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13076923787593842, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05025641247630119, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.009040200151503086, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8018749356269836, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.09861075133085251, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11231470853090286, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.09617619216442108, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.07963181287050247, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4064994156360626, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7568735480308533, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3421257734298706, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4391723573207855, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3349066972732544, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3250134587287903, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5801416635513306, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7360937595367432, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4482964873313904, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4150802791118622, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.37386658787727356, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5905559062957764, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7961419224739075, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.38867077231407166, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46345123648643494, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.40540412068367004, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43241727352142334, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6809003949165344, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44829487800598145, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.36865490674972534, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3481667935848236, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6698432564735413, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7080509066581726, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4432741403579712, "metrics/hit_grouping/tracker/all/object_completeness": 0.3913121819496155, "metrics/hit_grouping/tracker/all/object_purity": 0.3611101806163788, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.624787449836731, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00852200761437416, "metrics/hit_grouping/tracker/background_recall": 0.0354202464222908, "metrics/hit_grouping/tracker/completeness": 0.4138995409011841, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5653200149536133, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7189276814460754, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4454669654369354, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4454834759235382, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.40638938546180725, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6184965968132019, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6986454725265503, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46049875020980835, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3754051923751831, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.35151416063308716, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6381115317344666, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6938990354537964, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4201051890850067, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.28796055912971497, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.27079227566719055, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6245954632759094, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7323716878890991, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.47340044379234314, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4221530258655548, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3843531310558319, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.585729718208313, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7440252304077148, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45472702383995056, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41062361001968384, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4002305269241333, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5685752034187317, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.670041561126709, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4020135700702667, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.34773895144462585, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3239518404006958, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6855537295341492, "metrics/hit_grouping/tracker/purity": 0.3959601819515228, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8860819339752197, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012163336388766766, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.018668251112103462, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.01903681270778179, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06385751813650131, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6736382842063904, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.16018661856651306, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1432119607925415, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1473604142665863, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5785381197929382, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6877778172492981, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5340889096260071, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47019845247268677, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4319309890270233, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7062747478485107, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.43614184856414795, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5166969299316406, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5655745267868042, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5734599828720093, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0495854914188385, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04265521466732025, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.196256637573242, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26143109798431396, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.13099130988121033, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7430154085159302, "metrics/particle/class_1/pid_accuracy": 0.935369610786438, "metrics/particle/class_2/efficiency": 0.34720665216445923, "metrics/particle/class_2/pid_accuracy": 0.3890145421028137, "metrics/particle/class_3/efficiency": 0.534292995929718, "metrics/particle/class_3/pid_accuracy": 0.9286087155342102, "metrics/particle/class_4/efficiency": 0.6179018020629883, "metrics/particle/class_4/pid_accuracy": 0.1993769407272339, "metrics/particle/class_5/efficiency": 0.800000011920929, "metrics/particle/class_5/pid_accuracy": 0.29499998688697815, "metrics/particle/count/bias_mean": -8.767578125, "metrics/particle/count/mae": 9.572265625, "metrics/particle/count/prediction_mean": 61.28841018676758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.10687495023012161, "metrics/particle/event/energy_response_mean": 0.9595028758049011, "metrics/particle/event/met_abs_error": 11.124991416931152, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08522781729698181, "metrics/particle/event/scalar_pt_response_mean": 0.9728267788887024, "metrics/particle/event/vector_pt_closure": 0.0886974111199379, "metrics/particle/matched/delta_eta_abs_mean": 0.01753460057079792, "metrics/particle/matched/delta_phi_abs_mean": 0.017761046066880226, "metrics/particle/matched/delta_r_mean": 0.027923356741666794, "metrics/particle/matched/energy_relative_abs_mean": 0.17965814471244812, "metrics/particle/matched/pid_accuracy": 0.896507203578949, "metrics/particle/matched/pt_relative_abs_mean": 0.17570963501930237, "metrics/particle/matching/duplicate_fraction": 0.09300077706575394, "metrics/particle/matching/efficiency": 0.6188688278198242, "metrics/particle/matching/f1": 0.6601799130439758, "metrics/particle/matching/purity": 0.7074007391929626, "metrics/particle/matching/target_coverage_dr0p05": 0.5721428394317627, "metrics/particle/matching/target_coverage_dr0p10": 0.6902589201927185, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.40985631942749023, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6688660383224487, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02584698610007763, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1741376519203186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17150633037090302, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01276179775595665, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2120078057050705, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013864043168723583, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.047552227973938, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.010934105142951012, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.669257402420044, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6708.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3437407612800598, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4416598081588745, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7130333185195923, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5421885251998901, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20569585263729095, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.676310539245605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.026031816378235817, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5376166105270386, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.1388639658689499, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.933366537094116, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8989.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6133869886398315, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.008040860295295715, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.0963716506958, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.857955455780029, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.4169031381607056, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.9072403907775879, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.16494369506836, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.4043054580688477, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.05154800415039, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025728382170200348, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18055683374404907, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17458121478557587, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016705192625522614, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21640169620513916, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03624595329165459, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.826809048652649, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.21702665090560913, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.423518180847168, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1545.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.41637587547302246, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9219896197319031, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.395505666732788, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.224090099334717, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21200387179851532, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.933549880981445, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05759922415018082, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6774234771728516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11522412300109863, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.5797038078308105, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2066.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5114200711250305, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -10.936046600341797, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.629900932312012, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8530888557434082, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.2713785767555237, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.18566001951694489, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.413026809692383, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.2692466378211975, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.07086753845215, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028822144493460655, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1813121736049652, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17679908871650696, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008608303032815456, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21703557670116425, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0018310993909835815, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3721024990081787, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.014027591794729233, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0349345207214355, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13653.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3917216956615448, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.4155983924865723, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.9226897954940796, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7443463802337646, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2400469034910202, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.04709243774414, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.009991333819925785, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7806499600410461, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03563051298260689, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.829993963241577, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19617.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6368895173072815, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.31631678342819214, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.0253267288208, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.02380657196045, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -3.0918214321136475, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.835615634918213, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.930379867553711, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.2868518829345703, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.71543025970459, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027692900970578194, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1790619194507599, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1750219315290451, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010451238602399826, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21546347439289093, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007943029515445232, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6815354824066162, "metrics/particle/momentum/all/accepted/pz_bias": -0.027397554367780685, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0565528869628906, "metrics/particle/momentum/all/accepted/samples": 21906.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3793206512928009, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.0295612812042236, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.8930939435958862, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.7174148559570312, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.22809076309204102, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.242525100708008, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017899060621857643, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1302556991577148, "metrics/particle/momentum/all/assigned/pz_bias": 0.07124624401330948, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6778078079223633, "metrics/particle/momentum/all/assigned/samples": 30672.0, "metrics/particle/momentum/all/efficiency": 0.6189184784889221, "metrics/particle/momentum/all/event/energy_bias": -10.627771377563477, "metrics/particle/momentum/all/event/p3_error_mean": 29.24074363708496, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.193183898925781, "metrics/particle/momentum/all/event/px_bias": -4.7801032066345215, "metrics/particle/momentum/all/event/py_bias": -5.928515911102295, "metrics/particle/momentum/all/event/pz_abs_mean": 20.979780197143555, "metrics/particle/momentum/all/event/pz_bias": 3.421910524368286, "metrics/particle/momentum/all/event/pz_residual_rms": 33.8257942199707, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030765952542424202, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.18078340590000153, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17723770439624786, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015396117232739925, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21715569496154785, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007630521897226572, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7077702879905701, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014107952825725079, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.016084671020508, "metrics/particle/momentum/class_1/accepted/samples": 12450.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2020650953054428, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6518515944480896, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7312844395637512, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5601736903190613, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.15473826229572296, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.3333611488342285, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01249439362436533, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9435203075408936, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01828767918050289, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8970210552215576, "metrics/particle/momentum/class_1/assigned/samples": 15607.0, "metrics/particle/momentum/class_1/efficiency": 0.7460897564888, "metrics/particle/momentum/class_1/event/energy_bias": 12.003302574157715, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.796772003173828, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.94262981414795, "metrics/particle/momentum/class_1/event/px_bias": -1.0324363708496094, "metrics/particle/momentum/class_1/event/py_bias": -3.1364057064056396, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.715658187866211, "metrics/particle/momentum/class_1/event/pz_bias": -0.6637527346611023, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.232057571411133, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03931371122598648, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2147497534751892, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2120727300643921, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0661865621805191, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2514539659023285, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0430000014603138, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0962977409362793, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.23225072026252747, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.872686862945557, "metrics/particle/momentum/class_2/accepted/samples": 1000.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6564715504646301, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8490826487541199, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0258947610855103, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6365669965744019, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.334237664937973, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.700178623199463, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06077048182487488, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8953123092651367, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.31047773361206055, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.010106086730957, "metrics/particle/momentum/class_2/assigned/samples": 1843.0, "metrics/particle/momentum/class_2/efficiency": 0.3359086215496063, "metrics/particle/momentum/class_2/event/energy_bias": -22.627031326293945, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.95600700378418, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.353865623474121, "metrics/particle/momentum/class_2/event/px_bias": -0.8932639360427856, "metrics/particle/momentum/class_2/event/py_bias": -1.7626430988311768, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.32437515258789, "metrics/particle/momentum/class_2/event/pz_bias": 2.3161888122558594, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.453916549682617, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021400252357125282, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1709461808204651, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1661032736301422, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013588027097284794, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2065657526254654, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002723109209910035, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.41543346643447876, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009853837080299854, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2996947765350342, "metrics/particle/momentum/class_3/accepted/samples": 8079.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5612201690673828, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.8923990726470947, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.4359123706817627, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.28647780418396, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3047095835208893, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.78653335571289, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.017503924667835236, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0780019760131836, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.10880686342716217, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.824410915374756, "metrics/particle/momentum/class_3/assigned/samples": 12740.0, "metrics/particle/momentum/class_3/efficiency": 0.5312684774398804, "metrics/particle/momentum/class_3/event/energy_bias": 4.689972877502441, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.3570556640625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.150409698486328, "metrics/particle/momentum/class_3/event/px_bias": -2.4417994022369385, "metrics/particle/momentum/class_3/event/py_bias": -1.352739691734314, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.538796424865723, "metrics/particle/momentum/class_3/event/pz_bias": 1.7262954711914062, "metrics/particle/momentum/class_3/event/pz_residual_rms": 22.04024887084961, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.032118115574121475, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18597450852394104, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1795831173658371, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.008840872906148434, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22850634157657623, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016806723549962044, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.285760760307312, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03096049651503563, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6595046520233154, "metrics/particle/momentum/class_4/accepted/samples": 238.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3471153974533081, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6815974712371826, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.88498592376709, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.72011661529541, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21779140830039978, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.878021240234375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.015337423421442509, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.230134129524231, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.013210948556661606, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.52618408203125, "metrics/particle/momentum/class_4/assigned/samples": 326.0, "metrics/particle/momentum/class_4/efficiency": 0.6538461446762085, "metrics/particle/momentum/class_4/event/energy_bias": -2.8130998611450195, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.7873551845550537, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7900142669677734, "metrics/particle/momentum/class_4/event/px_bias": 0.0908883735537529, "metrics/particle/momentum/class_4/event/py_bias": 0.3560638427734375, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0696613788604736, "metrics/particle/momentum/class_4/event/pz_bias": 0.2195691466331482, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.70698356628418, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.027008144184947014, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22799381613731384, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22056815028190613, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.060662247240543365, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2613367736339569, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07194244861602783, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.585465669631958, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.8635649681091309, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.623743057250977, "metrics/particle/momentum/class_5/accepted/samples": 139.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05070831626653671, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3143555223941803, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.30171579122543335, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.027197260409593582, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07692307978868484, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6072747707366943, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.0182785987854004, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3477819263935089, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.608626842498779, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8742138147354126, "metrics/particle/momentum/class_5/event/energy_bias": -1.880915641784668, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.2912018299102783, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.483888864517212, "metrics/particle/momentum/class_5/event/px_bias": -0.5034916400909424, "metrics/particle/momentum/class_5/event/py_bias": -0.0327908918261528, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7909680604934692, "metrics/particle/momentum/class_5/event/pz_bias": -0.17639029026031494, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.258523464202881, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5288668274879456, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5285423398017883, "metrics/set_slot/angular_recall_all_dr0p10": 0.887729287147522, "metrics/set_slot/angular_recall_selected_dr0p10": 0.700890302658081, "metrics/set_slot/soft_count_bias_mean": 0.7196791768074036}, "step_train_time": 4.534199953079224, "step_valid_time": 27809.35736489296, "step_total_time": 27813.891565322876, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9918973802413524, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21410680925568304, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2987, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2746, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2675, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9071688140072679, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9193170405088718, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9132025274359827, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08068295949112822, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8837132474397092, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8955473719450954, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8895909544396409, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9071688140072679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9783733380797346, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2105834741491599, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2048, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2028, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.905393457117595, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9171518137035378, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9112347052280312, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08284818629646216, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.896551724137931, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9081952530228392, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9023359288097886, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.905393457117595, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9952224676875369, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22460423490389392, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1256, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1127, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1102, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.875, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8972929936305732, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8860062893081762, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10270700636942676, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8555900621118012, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8773885350318471, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8663522012578616, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.875} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_12000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..45178f2420b87abb1eacc51f36b79d24133a88fd --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5685048322910745, "train": {"Classification_binary": 0.3147865831851959, "Cardinality": 0.264827698469162, "Classification": 0.14869865775108337, "Regression_pt": 0.24378523230552673, "Regression_eta": 0.002825002418830991, "Regression_sin_phi": 0.0014596593100577593, "Regression_cos_phi": 0.0015416030073538423, "Regression_energy": 0.24333558976650238, "HitGrouping": 0.23208047449588776, "Auxiliary": 0.3760436177253723, "Total": 1.8417037725448608, "Total_unweighted": 0.956432327744551}, "train_group": {}, "valid": {"Classification_binary": 0.2645710309346517, "Cardinality": 0.19662100076675415, "Classification": 0.13468945026397705, "Regression_pt": 0.2301058570543925, "Regression_eta": 0.0026427069678902626, "Regression_sin_phi": 0.001411021842310826, "Regression_cos_phi": 0.0014547728933393955, "Regression_energy": 0.230151096979777, "HitGrouping": 0.22961103916168213, "Auxiliary": 0.3321775197982788, "Total": 1.6388325691223145, "Total_unweighted": 0.8650259369363387, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7741761803627014, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.34046506881713867, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4065363109111786, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.33429330587387085, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.32314741611480713, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5399074554443359, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7225692868232727, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.30289459228515625, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.401897668838501, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2959379553794861, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.307486355304718, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6147068738937378, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7898690104484558, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.34151920676231384, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.38255345821380615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3363124132156372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3194206953048706, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5135316848754883, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7787826657295227, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3377155363559723, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3920060694217682, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.33204126358032227, "metrics/hit_grouping/calorimeter/all/object_purity": 0.31954947113990784, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5310942530632019, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02532656118273735, "metrics/hit_grouping/calorimeter/background_recall": 0.6246229410171509, "metrics/hit_grouping/calorimeter/completeness": 0.41154545545578003, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4418039917945862, "metrics/hit_grouping/calorimeter/energy_purity": 0.4365478754043579, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5274116396903992, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7981261014938354, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3442821204662323, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3252314031124115, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.34056758880615234, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3299029767513275, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4864138662815094, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7717870473861694, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3612620234489441, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.43461912870407104, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.35440143942832947, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3471185564994812, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5487627387046814, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7451291084289551, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2992785573005676, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.48893946409225464, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.29069241881370544, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.26890483498573303, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6072556972503662, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7605382800102234, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3747773766517639, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4827779531478882, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3696894347667694, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.35415515303611755, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5684833526611328, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7706611752510071, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4869312047958374, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.645211935043335, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47510406374931335, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4347573518753052, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5379382371902466, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7994357347488403, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.278777539730072, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.262019544839859, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2732603847980499, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2676037549972534, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.49002277851104736, "metrics/hit_grouping/calorimeter/purity": 0.40529152750968933, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.901103138923645, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05348131060600281, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14769230782985687, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05528205260634422, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012027797289192677, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8145838379859924, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.10385316610336304, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10490307956933975, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.10236791521310806, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.08759287744760513, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3665906488895416, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7741163372993469, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.35722455382347107, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4137263894081116, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3511500060558319, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.33944520354270935, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5510558485984802, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7384823560714722, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4426180422306061, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4271528124809265, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4003155529499054, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5878661274909973, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7972421050071716, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3982052803039551, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48069995641708374, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4217833876609802, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43241727352142334, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6813099980354309, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4345908463001251, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37569910287857056, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3635276257991791, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6720485091209412, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.709144115447998, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4340638518333435, "metrics/hit_grouping/tracker/all/object_completeness": 0.40080514550209045, "metrics/hit_grouping/tracker/all/object_purity": 0.38006144762039185, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6252597570419312, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004132899455726147, "metrics/hit_grouping/tracker/background_recall": 0.025765562430024147, "metrics/hit_grouping/tracker/completeness": 0.42420586943626404, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5535733103752136, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7186429500579834, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4321640133857727, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45583075284957886, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4349537789821625, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6203188896179199, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7011793851852417, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44717302918434143, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3843529224395752, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3677380681037903, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6358271241188049, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6965204477310181, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.424352765083313, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2961363196372986, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.27130183577537537, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.625, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7328933477401733, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45929187536239624, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4279489815235138, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4009860157966614, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5877965688705444, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.750493049621582, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45938748121261597, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4193384349346161, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4024111330509186, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5499334335327148, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6709343791007996, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.396745502948761, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3621850311756134, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3488723933696747, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.687309741973877, "metrics/hit_grouping/tracker/purity": 0.4040312170982361, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8939002156257629, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014335360378026962, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.019599715247750282, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02063230611383915, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08079930394887924, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6881583333015442, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1679626703262329, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.18034720420837402, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.17470449209213257, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5629860162734985, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6864283680915833, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5213977694511414, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4777072072029114, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4522850215435028, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7063337564468384, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.42292335629463196, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5405942797660828, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5960158705711365, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5821106433868408, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0628039538860321, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05130590498447418, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.27271270751953, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.268347829580307, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.13790801167488098, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7548872232437134, "metrics/particle/class_1/pid_accuracy": 0.9483644366264343, "metrics/particle/class_2/efficiency": 0.3128786087036133, "metrics/particle/class_2/pid_accuracy": 0.3997848629951477, "metrics/particle/class_3/efficiency": 0.5332418084144592, "metrics/particle/class_3/pid_accuracy": 0.9338590502738953, "metrics/particle/class_4/efficiency": 0.6265640258789062, "metrics/particle/class_4/pid_accuracy": 0.24423962831497192, "metrics/particle/class_5/efficiency": 0.7540000081062317, "metrics/particle/class_5/pid_accuracy": 0.2785145938396454, "metrics/particle/count/bias_mean": -11.445963859558105, "metrics/particle/count/mae": 11.748046875, "metrics/particle/count/prediction_mean": 58.61002731323242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.10674494504928589, "metrics/particle/event/energy_response_mean": 0.9888706803321838, "metrics/particle/event/met_abs_error": 10.901935577392578, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09021397680044174, "metrics/particle/event/scalar_pt_response_mean": 1.0115278959274292, "metrics/particle/event/vector_pt_closure": 0.0858306884765625, "metrics/particle/matched/delta_eta_abs_mean": 0.018538841977715492, "metrics/particle/matched/delta_phi_abs_mean": 0.016945483162999153, "metrics/particle/matched/delta_r_mean": 0.028084170073270798, "metrics/particle/matched/energy_relative_abs_mean": 0.17676019668579102, "metrics/particle/matched/pid_accuracy": 0.9094350934028625, "metrics/particle/matched/pt_relative_abs_mean": 0.17371706664562225, "metrics/particle/matching/duplicate_fraction": 0.08556511998176575, "metrics/particle/matching/efficiency": 0.6210155487060547, "metrics/particle/matching/f1": 0.6762602925300598, "metrics/particle/matching/purity": 0.7422938346862793, "metrics/particle/matching/target_coverage_dr0p05": 0.5742616653442383, "metrics/particle/matching/target_coverage_dr0p10": 0.6875545978546143, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4121238589286804, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6641823053359985, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026077592745423317, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17330536246299744, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17007455229759216, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027444591745734215, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21100781857967377, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011678394861519337, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9675533771514893, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01939212903380394, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3805882930755615, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6679.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28379637002944946, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5065463781356812, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.86074697971344, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7059587240219116, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17762696743011475, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.09002113342285, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02714720368385315, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.4907866716384888, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.022953692823648453, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.791249990463257, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8546.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6107351779937744, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0441750288009644, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.501814842224121, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.825897216796875, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.678935706615448, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.09048725664615631, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.582637786865234, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7478152513504028, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.848649978637695, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027415480464696884, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18145303428173065, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17714066803455353, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028545621782541275, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21948091685771942, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03689320385456085, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.780950903892517, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12986882030963898, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.08201265335083, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1545.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.33359572291374207, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.974393367767334, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.6298670768737793, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.477738618850708, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18944723904132843, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.328292846679688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06482411921024323, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6031198501586914, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.10622425377368927, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.175939559936523, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1990.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5114200711250305, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.130050659179688, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.732051849365234, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8590644598007202, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11549775302410126, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0428178571164608, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.527162551879883, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.3125009536743164, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.517311096191406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028399528935551643, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1753055304288864, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17320846021175385, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02123660035431385, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21339964866638184, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0014590020291507244, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3491969406604767, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0017151032807305455, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9758904576301575, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13708.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3432706296443939, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.467428684234619, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.0101706981658936, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.8464456796646118, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.21906618773937225, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.724706649780273, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011540893465280533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7779722809791565, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.018531663343310356, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.968656063079834, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 18976.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6394551396369934, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.288537979125977, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.885659217834473, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.809630393981934, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.997187852859497, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.15242153406143188, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.127511978149414, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.33012548089027405, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.771705627441406, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02762310579419136, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1751294732093811, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1725310981273651, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02364201657474041, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21310964226722717, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007067298982292414, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6383663415908813, "metrics/particle/momentum/all/accepted/pz_bias": -0.013982157222926617, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.869466781616211, "metrics/particle/momentum/all/accepted/samples": 21932.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3253958523273468, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.088503122329712, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.0086874961853027, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.848332166671753, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.20506912469863892, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.946632385253906, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.019653022289276123, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1074570417404175, "metrics/particle/momentum/all/assigned/pz_bias": -0.012431582435965538, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5214827060699463, "metrics/particle/momentum/all/assigned/samples": 29512.0, "metrics/particle/momentum/all/efficiency": 0.619653046131134, "metrics/particle/momentum/all/event/energy_bias": -1.7973380088806152, "metrics/particle/momentum/all/event/p3_error_mean": 29.38360595703125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.959344863891602, "metrics/particle/momentum/all/event/px_bias": -2.791620969772339, "metrics/particle/momentum/all/event/py_bias": -0.20009079575538635, "metrics/particle/momentum/all/event/pz_abs_mean": 21.57211685180664, "metrics/particle/momentum/all/event/pz_bias": -2.7301905155181885, "metrics/particle/momentum/all/event/pz_residual_rms": 33.429691314697266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030974870547652245, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1782008856534958, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1721520721912384, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.019088104367256165, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21200884878635406, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0068156602792441845, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6633014678955078, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03155094385147095, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7736546993255615, "metrics/particle/momentum/class_1/accepted/samples": 12618.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18531598150730133, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8323575258255005, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.9859575629234314, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8287046551704407, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.14002984762191772, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.083622932434082, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014210628345608711, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9355961680412292, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.043146081268787384, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.776383399963379, "metrics/particle/momentum/class_1/assigned/samples": 15411.0, "metrics/particle/momentum/class_1/efficiency": 0.7561575174331665, "metrics/particle/momentum/class_1/event/energy_bias": 19.45323944091797, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.93474769592285, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.00378131866455, "metrics/particle/momentum/class_1/event/px_bias": -1.016280174255371, "metrics/particle/momentum/class_1/event/py_bias": 2.603414297103882, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.033382415771484, "metrics/particle/momentum/class_1/event/pz_bias": -5.022906303405762, "metrics/particle/momentum/class_1/event/pz_residual_rms": 25.083309173583984, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03783651813864708, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21720917522907257, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21120889484882355, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.041512370109558105, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2525251507759094, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0416666679084301, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.052262783050537, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13052549958229065, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.556546211242676, "metrics/particle/momentum/class_2/accepted/samples": 888.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6131433248519897, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8724839687347412, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2874667644500732, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.9178959727287292, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33066993951797485, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 15.471735000610352, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06699446588754654, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.038007974624634, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11565081030130386, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.727826118469238, "metrics/particle/momentum/class_2/assigned/samples": 1627.0, "metrics/particle/momentum/class_2/efficiency": 0.2982868552207947, "metrics/particle/momentum/class_2/event/energy_bias": -19.545204162597656, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.941905975341797, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.625065803527832, "metrics/particle/momentum/class_2/event/px_bias": -0.9565833210945129, "metrics/particle/momentum/class_2/event/py_bias": -2.594390630722046, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.1586856842041, "metrics/particle/momentum/class_2/event/pz_bias": 0.8806442022323608, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.625741958618164, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021153593435883522, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16530980169773102, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16860069334506989, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03837576135993004, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20988066494464874, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002847591880708933, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.40885835886001587, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0020246708299964666, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3870558738708496, "metrics/particle/momentum/class_3/accepted/samples": 8077.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4684706926345825, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.890228033065796, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.450674533843994, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.3155367374420166, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2714250087738037, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.507122039794922, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.019488630816340446, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0404962301254272, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01752971112728119, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.728388547897339, "metrics/particle/momentum/class_3/assigned/samples": 12007.0, "metrics/particle/momentum/class_3/efficiency": 0.5311369895935059, "metrics/particle/momentum/class_3/event/energy_bias": 1.1997255086898804, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.115800857543945, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.977025985717773, "metrics/particle/momentum/class_3/event/px_bias": -0.713310956954956, "metrics/particle/momentum/class_3/event/py_bias": -0.8182340860366821, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.560173988342285, "metrics/particle/momentum/class_3/event/pz_bias": 1.2946456670761108, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.1313533782959, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.031685858964920044, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1836591362953186, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17945228517055511, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04375888407230377, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21863442659378052, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.021834060549736023, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.298466682434082, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17569699883460999, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.7259788513183594, "metrics/particle/momentum/class_4/accepted/samples": 229.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.35678166151046753, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9948824644088745, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.616114854812622, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.453904151916504, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21766561269760132, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.939830780029297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.015772869810461998, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3034679889678955, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.14395850896835327, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.6206376552581787, "metrics/particle/momentum/class_4/assigned/samples": 317.0, "metrics/particle/momentum/class_4/efficiency": 0.6291208863258362, "metrics/particle/momentum/class_4/event/energy_bias": -1.4800770282745361, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.968759059906006, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.842278003692627, "metrics/particle/momentum/class_4/event/px_bias": 0.29460409283638, "metrics/particle/momentum/class_4/event/py_bias": 0.1688406765460968, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.266848564147949, "metrics/particle/momentum/class_4/event/pz_bias": 0.4876370429992676, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.504595756530762, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.027304651215672493, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1854468286037445, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1775101274251938, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.045464858412742615, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2149118185043335, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03333333507180214, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.741706132888794, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6753450036048889, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.263371706008911, "metrics/particle/momentum/class_5/accepted/samples": 120.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07710625231266022, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3104915916919708, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.31014174222946167, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08572948724031448, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.18666666746139526, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5391934514045715, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08666666597127914, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.7701644897460938, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.9748963713645935, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.447933197021484, "metrics/particle/momentum/class_5/assigned/samples": 150.0, "metrics/particle/momentum/class_5/efficiency": 0.7547169923782349, "metrics/particle/momentum/class_5/event/energy_bias": -1.425021767616272, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.5159616470336914, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.7561123371124268, "metrics/particle/momentum/class_5/event/px_bias": -0.4000507593154907, "metrics/particle/momentum/class_5/event/py_bias": 0.4402787685394287, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.74446702003479, "metrics/particle/momentum/class_5/event/pz_bias": -0.3702108860015869, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.117589473724365, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5126394629478455, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.512707531452179, "metrics/set_slot/angular_recall_all_dr0p10": 0.8993178606033325, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6999145150184631, "metrics/set_slot/soft_count_bias_mean": -2.260735273361206}, "step_train_time": 5.053609371185303, "step_valid_time": 30389.061322450638, "step_total_time": 30394.11493229866, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0384019744728148, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22781079944663618, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2766, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2676, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9137760158572844, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9269436997319035, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9203127599401097, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07305630026809651, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8840436075322101, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8967828418230563, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8903676592912992, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9137760158572844, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0093947189336516, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2103969389085798, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2066, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2046, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9133510167992926, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9260421335723891, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9196527932339195, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07395786642761093, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9045092838196287, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9170775437023756, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.91075005564211, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9133510167992926, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0249393806605407, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22229790784538872, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1250, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1135, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1117, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8812111801242236, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.908, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8944050433412135, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.092, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8672360248447205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8936, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.880220646178093, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8812111801242236} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_13000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a57090dd881bc3ca829bd6878cfa7b599a0eda0 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6158802349819974, "train": {"Classification_binary": 0.29902538657188416, "Cardinality": 0.2713960111141205, "Classification": 0.13820300996303558, "Regression_pt": 0.21992595493793488, "Regression_eta": 0.0028304855804890394, "Regression_sin_phi": 0.0014913398772478104, "Regression_cos_phi": 0.0014749930705875158, "Regression_energy": 0.2162085920572281, "HitGrouping": 0.22880104184150696, "Auxiliary": 0.35469216108322144, "Total": 1.7741070985794067, "Total_unweighted": 0.8791597620584071}, "train_group": {}, "valid": {"Classification_binary": 0.2568756143252055, "Cardinality": 0.1985632578531901, "Classification": 0.12617347637812296, "Regression_pt": 0.21946998437245688, "Regression_eta": 0.0025099323441584906, "Regression_sin_phi": 0.001388061015556256, "Regression_cos_phi": 0.0013432675041258335, "Regression_energy": 0.21732640266418457, "HitGrouping": 0.22694973150889078, "Auxiliary": 0.3128097852071126, "Total": 1.5796343485514324, "Total_unweighted": 0.8250867386038105, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.779708743095398, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3550959527492523, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41758760809898376, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.34879469871520996, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3343965709209442, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5247945785522461, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7341257333755493, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3360261917114258, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4046086072921753, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.33047497272491455, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3264710009098053, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5818366408348083, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7926605343818665, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3571683168411255, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3809007406234741, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3513965606689453, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.33349207043647766, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5037186145782471, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7832423448562622, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.354605108499527, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3948328197002411, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.348682701587677, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3331533670425415, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5175076127052307, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025265896692872047, "metrics/hit_grouping/calorimeter/background_recall": 0.625196635723114, "metrics/hit_grouping/calorimeter/completeness": 0.42369866371154785, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4511202275753021, "metrics/hit_grouping/calorimeter/energy_purity": 0.4457132816314697, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5064550042152405, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8059132099151611, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3642726540565491, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3274410367012024, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3601801097393036, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3446521759033203, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.46234697103500366, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7752746343612671, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3756980001926422, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.434133917093277, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36895421147346497, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3570616841316223, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5439107418060303, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.743558943271637, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3121612071990967, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49652382731437683, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3032217025756836, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2838994264602661, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6067500710487366, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7651017904281616, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3878623843193054, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4785425066947937, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3818739354610443, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.36909663677215576, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5601370334625244, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.769967794418335, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.490508109331131, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5944531559944153, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.48090121150016785, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4486181437969208, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5363683700561523, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8044432997703552, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.30146077275276184, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.27929556369781494, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.29608094692230225, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.27973678708076477, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.46926477551460266, "metrics/hit_grouping/calorimeter/purity": 0.4172340929508209, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.871418833732605, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0715729370713234, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15076923370361328, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06890843063592911, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02480851672589779, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05692307651042938, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.808859646320343, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12057384848594666, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10718358308076859, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.11786676943302155, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.09610436856746674, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3797035217285156, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7798896431922913, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.37409815192222595, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4165792167186737, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.36792147159576416, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.353359580039978, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5352505445480347, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7414848208427429, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43529587984085083, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.43611279129981995, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4099932610988617, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5862223505973816, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7998180389404297, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.36567583680152893, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4952530264854431, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.44351521134376526, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4094223082065582, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.69038987159729, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4311254620552063, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37968072295188904, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.35655930638313293, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6574781537055969, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.71575528383255, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42693179845809937, "metrics/hit_grouping/tracker/all/object_completeness": 0.4072505533695221, "metrics/hit_grouping/tracker/all/object_purity": 0.38077157735824585, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6140657663345337, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004495862405747175, "metrics/hit_grouping/tracker/background_recall": 0.019131671637296677, "metrics/hit_grouping/tracker/completeness": 0.43003273010253906, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5495609045028687, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.722133994102478, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4206833839416504, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46145278215408325, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.43821263313293457, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6104783415794373, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7087103724479675, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4448886215686798, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38929271697998047, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3648841679096222, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6249762177467346, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.709080696105957, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42172330617904663, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3060093820095062, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.27014055848121643, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6104369163513184, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7426548004150391, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45551761984825134, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43422508239746094, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.39807549118995667, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5741373300552368, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7704366445541382, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4347536563873291, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4390578866004944, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4049757122993469, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5039947032928467, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6710972189903259, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3883165419101715, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3665168881416321, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.35397303104400635, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6854366660118103, "metrics/hit_grouping/tracker/purity": 0.40973028540611267, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8942777514457703, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.006950478069484234, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.019445031881332397, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.023523200303316116, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07298001646995544, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6838768124580383, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.18040435016155243, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1915564388036728, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.18543866276741028, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5697252750396729, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6951269507408142, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5120713114738464, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4845069944858551, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4515574276447296, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.692639172077179, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.41477513313293457, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5472515225410461, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6053953170776367, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5879712104797363, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0709521695971489, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.057166434824466705, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.400218963623047, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28361716866493225, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15317733585834503, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7875543832778931, "metrics/particle/class_1/pid_accuracy": 0.9511594772338867, "metrics/particle/class_2/efficiency": 0.33957818150520325, "metrics/particle/class_2/pid_accuracy": 0.47571852803230286, "metrics/particle/class_3/efficiency": 0.5368459224700928, "metrics/particle/class_3/pid_accuracy": 0.940497100353241, "metrics/particle/class_4/efficiency": 0.6660250425338745, "metrics/particle/class_4/pid_accuracy": 0.23121386766433716, "metrics/particle/class_5/efficiency": 0.8479999899864197, "metrics/particle/class_5/pid_accuracy": 0.33726415038108826, "metrics/particle/count/bias_mean": -11.166666984558105, "metrics/particle/count/mae": 11.453125, "metrics/particle/count/prediction_mean": 58.88932418823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1051078736782074, "metrics/particle/event/energy_response_mean": 0.9467008709907532, "metrics/particle/event/met_abs_error": 11.4856595993042, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09206026792526245, "metrics/particle/event/scalar_pt_response_mean": 0.9469056129455566, "metrics/particle/event/vector_pt_closure": 0.08665808290243149, "metrics/particle/matched/delta_eta_abs_mean": 0.016620460897684097, "metrics/particle/matched/delta_phi_abs_mean": 0.016950175166130066, "metrics/particle/matched/delta_r_mean": 0.026636283844709396, "metrics/particle/matched/energy_relative_abs_mean": 0.17044517397880554, "metrics/particle/matched/pid_accuracy": 0.9154269695281982, "metrics/particle/matched/pt_relative_abs_mean": 0.16641201078891754, "metrics/particle/matching/duplicate_fraction": 0.08141154795885086, "metrics/particle/matching/efficiency": 0.6409493684768677, "metrics/particle/matching/f1": 0.6964555978775024, "metrics/particle/matching/purity": 0.76248699426651, "metrics/particle/matching/target_coverage_dr0p05": 0.586714506149292, "metrics/particle/matching/target_coverage_dr0p10": 0.697637677192688, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.43926918506622314, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.683837354183197, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024245696142315865, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1625710427761078, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.160456582903862, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005122478120028973, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20079131424427032, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012334929779171944, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9596024751663208, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.11918772757053375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.5195276737213135, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6891.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24465861916542053, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.346897006034851, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.398748517036438, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.228169560432434, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15418656170368195, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.269798278808594, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022941656410694122, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3820000886917114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.21660125255584717, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9103786945343018, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8587.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6301206946372986, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -5.428928852081299, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.211997985839844, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.504733562469482, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6815098524093628, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.544208288192749, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.396537780761719, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.8824048042297363, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.835617065429688, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023831482976675034, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17302028834819794, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17316757142543793, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025027122348546982, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21446943283081055, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.036305733025074005, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7944605350494385, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.3633544147014618, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.366908073425293, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1570.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.27007055282592773, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2286863327026367, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.0256123542785645, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.861542224884033, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1637100875377655, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 50.86071014404297, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06284845620393753, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6190412044525146, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.6772047281265259, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.807453632354736, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1973.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5196954607963562, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.000264167785645, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.669995307922363, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8087167739868164, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08564206957817078, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.44984081387519836, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.452863693237305, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.18976402282714844, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 28.421981811523438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027115119621157646, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16996879875659943, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1649489551782608, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.002737676491960883, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20462438464164734, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0017583344597369432, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3369239866733551, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.024898642674088478, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9979914426803589, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14218.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3359440863132477, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.368645429611206, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.7282543182373047, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.5521055459976196, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.19677284359931946, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.957756996154785, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.010320620611310005, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7357970476150513, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.016616929322481155, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.9462673664093018, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19088.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6632457971572876, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.20678897202014923, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.557198524475098, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.614447593688965, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.3728151321411133, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 4.097452163696289, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.960804462432861, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.49282965064048767, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.719768524169922, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026015929877758026, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16793225705623627, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16415289044380188, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005005329381674528, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20416806638240814, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007363640237599611, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6270253658294678, "metrics/particle/momentum/all/accepted/pz_bias": 0.07697861641645432, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9680957794189453, "metrics/particle/momentum/all/accepted/samples": 22679.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.30512121319770813, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.9968534708023071, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7191548347473145, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.5454230308532715, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18223826587200165, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.417991638183594, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017471667379140854, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0482829809188843, "metrics/particle/momentum/all/assigned/pz_bias": 0.11849918961524963, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6195802688598633, "metrics/particle/momentum/all/assigned/samples": 29648.0, "metrics/particle/momentum/all/efficiency": 0.6407583355903625, "metrics/particle/momentum/all/event/energy_bias": -14.635982513427734, "metrics/particle/momentum/all/event/p3_error_mean": 30.44907569885254, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.809548377990723, "metrics/particle/momentum/all/event/px_bias": -3.1399667263031006, "metrics/particle/momentum/all/event/py_bias": 6.091501712799072, "metrics/particle/momentum/all/event/pz_abs_mean": 21.74043846130371, "metrics/particle/momentum/all/event/pz_bias": 3.1854708194732666, "metrics/particle/momentum/all/event/pz_residual_rms": 33.11052322387695, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028358742594718933, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16431604325771332, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1607276201248169, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0022179975640028715, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1998291015625, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0068902852945029736, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6430056691169739, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.09306980669498444, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.9238383769989014, "metrics/particle/momentum/class_1/accepted/samples": 13207.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16127833724021912, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6557122468948364, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.687318742275238, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5205379724502563, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11305855214595795, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.858306884765625, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.012519259937107563, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8612907528877258, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.130229189991951, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.896089792251587, "metrics/particle/momentum/class_1/assigned/samples": 15576.0, "metrics/particle/momentum/class_1/efficiency": 0.7914544343948364, "metrics/particle/momentum/class_1/event/energy_bias": 4.272212505340576, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.418609619140625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.620644569396973, "metrics/particle/momentum/class_1/event/px_bias": -0.9914478659629822, "metrics/particle/momentum/class_1/event/py_bias": 3.1484360694885254, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.822649002075195, "metrics/particle/momentum/class_1/event/pz_bias": 4.240466594696045, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.930683135986328, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03689957410097122, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20704416930675507, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20340664684772491, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03756486997008324, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.243162140250206, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03433923050761223, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8837534189224243, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.028925901278853416, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.364551067352295, "metrics/particle/momentum/class_2/accepted/samples": 961.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5439162254333496, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8070068359375, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8831178545951843, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.540930449962616, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30271902680397034, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.5878474712371826, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06102719157934189, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8478124141693115, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.25195157527923584, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.942265033721924, "metrics/particle/momentum/class_2/assigned/samples": 1655.0, "metrics/particle/momentum/class_2/efficiency": 0.3228082060813904, "metrics/particle/momentum/class_2/event/energy_bias": -15.983747482299805, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.43962287902832, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.829610824584961, "metrics/particle/momentum/class_2/event/px_bias": -0.8302463889122009, "metrics/particle/momentum/class_2/event/py_bias": 0.5591081976890564, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.056312561035156, "metrics/particle/momentum/class_2/event/pz_bias": 0.04407757520675659, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.866065979003906, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020909611135721207, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1679399162530899, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16388310492038727, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01457246020436287, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2048984318971634, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00332266790792346, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3972453474998474, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.025750579312443733, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.326017141342163, "metrics/particle/momentum/class_3/accepted/samples": 8126.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.46345803141593933, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.94035267829895, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.2028310298919678, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0442821979522705, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.25676241517066956, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.53057289123535, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016916505992412567, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.01132333278656, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07195925712585449, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.7726590633392334, "metrics/particle/momentum/class_3/assigned/samples": 11941.0, "metrics/particle/momentum/class_3/efficiency": 0.5343591570854187, "metrics/particle/momentum/class_3/event/energy_bias": 1.3103077411651611, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.420934677124023, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.723262786865234, "metrics/particle/momentum/class_3/event/px_bias": -1.3215304613113403, "metrics/particle/momentum/class_3/event/py_bias": 1.6084789037704468, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.794353485107422, "metrics/particle/momentum/class_3/event/pz_bias": -2.0664234161376953, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.398658752441406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02964894101023674, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19681712985038757, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18647319078445435, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05175299197435379, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22600865364074707, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.03629032149910927, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.4990195035934448, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.8433088660240173, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.845311403274536, "metrics/particle/momentum/class_4/accepted/samples": 248.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27375277876853943, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7298637628555298, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.598057746887207, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4501558542251587, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1894409954547882, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.558023452758789, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03105590119957924, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4463592767715454, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5905427932739258, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.604726552963257, "metrics/particle/momentum/class_4/assigned/samples": 322.0, "metrics/particle/momentum/class_4/efficiency": 0.6813187003135681, "metrics/particle/momentum/class_4/event/energy_bias": -1.769803762435913, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.405348777770996, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.204073429107666, "metrics/particle/momentum/class_4/event/px_bias": 0.14368095993995667, "metrics/particle/momentum/class_4/event/py_bias": 0.5801768898963928, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4405860900878906, "metrics/particle/momentum/class_4/event/pz_bias": 0.918124258518219, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.257702827453613, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020120088011026382, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18944114446640015, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19460298120975494, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07976597547531128, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2314801812171936, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.051094889640808105, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.321707010269165, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.514142632484436, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.441164016723633, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0758431926369667, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2920268774032593, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2772411108016968, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.020517291501164436, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09090909361839294, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.535229504108429, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06493506580591202, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.6555745601654053, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.11956994235515594, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.022946357727051, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.8616352081298828, "metrics/particle/momentum/class_5/event/energy_bias": -2.464951276779175, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.9815781116485596, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1871542930603027, "metrics/particle/momentum/class_5/event/px_bias": -0.140422984957695, "metrics/particle/momentum/class_5/event/py_bias": 0.19530130922794342, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6150120496749878, "metrics/particle/momentum/class_5/event/pz_bias": 0.04922584071755409, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.076131820678711, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.48737505078315735, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.48753637075424194, "metrics/set_slot/angular_recall_all_dr0p10": 0.9096890687942505, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7067263722419739, "metrics/set_slot/soft_count_bias_mean": -2.5012779235839844}, "step_train_time": 11.030357599258423, "step_valid_time": 33046.653371572495, "step_total_time": 33057.68372941017, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9861552061357185, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22773917958495382, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2974, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2773, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2708, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9160885365047903, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9324142568930733, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9241793034494251, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0675857431069267, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8946151304922365, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9105581708137189, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.902516247292118, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9160885365047903, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9477215612064451, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21018312968975628, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2221, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2060, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2050, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9106984969053935, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9275101305718145, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9190274369841623, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0724898694281855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9062776304155614, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9230076542098155, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9145661387463752, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9106984969053935, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9560330410829947, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2033540059748078, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1140, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8850931677018633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9156626506024096, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9001184366363996, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08433734939759036, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8734472049689441, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9036144578313253, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8882747729964469, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8850931677018633} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_14000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..79bdb04c1d5477425f1c22955e599cc164f2c894 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6632556376729202, "train": {"Classification_binary": 0.279066264629364, "Cardinality": 0.17570820450782776, "Classification": 0.1510295569896698, "Regression_pt": 0.20986714959144592, "Regression_eta": 0.002737667877227068, "Regression_sin_phi": 0.0015161697519943118, "Regression_cos_phi": 0.00143973168451339, "Regression_energy": 0.20828257501125336, "HitGrouping": 0.2257656306028366, "Auxiliary": 0.3277444541454315, "Total": 1.6232002973556519, "Total_unweighted": 0.8539391155354679}, "train_group": {}, "valid": {"Classification_binary": 0.2431915799776713, "Cardinality": 0.15596720576286316, "Classification": 0.12200964490572612, "Regression_pt": 0.1992136836051941, "Regression_eta": 0.0024925721809267998, "Regression_sin_phi": 0.0012470448079208534, "Regression_cos_phi": 0.0012317478346327941, "Regression_energy": 0.19725435972213745, "HitGrouping": 0.22056615352630615, "Auxiliary": 0.294366717338562, "Total": 1.4562950134277344, "Total_unweighted": 0.7666406330342094, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7744253277778625, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3717494010925293, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41711533069610596, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.36795440316200256, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.360152930021286, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5294228792190552, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7098844051361084, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3519250154495239, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4456116557121277, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3456534743309021, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3514491617679596, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6123347878456116, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7935241460800171, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3695373237133026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.386375367641449, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3653901219367981, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3509339690208435, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.49307921528816223, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.779876172542572, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.36866942048072815, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4015805423259735, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3644450008869171, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3539468050003052, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5154711008071899, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025552133098244667, "metrics/hit_grouping/calorimeter/background_recall": 0.6273742914199829, "metrics/hit_grouping/calorimeter/completeness": 0.4378727674484253, "metrics/hit_grouping/calorimeter/energy_completeness": 0.46356314420700073, "metrics/hit_grouping/calorimeter/energy_purity": 0.4580853283405304, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4991072416305542, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7975509762763977, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.379143625497818, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.330188125371933, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3773403465747833, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.36811068654060364, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.47046878933906555, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7760287523269653, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.38993412256240845, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4423823356628418, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.38329869508743286, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3789307475090027, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5297185778617859, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7464736700057983, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3243391811847687, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5101757049560547, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.31750237941741943, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29793089628219604, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5958791375160217, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7653995156288147, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4074361324310303, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48819682002067566, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4038360118865967, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.39764687418937683, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5496107339859009, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.770546019077301, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5057618618011475, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6352694630622864, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.49604979157447815, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4671613872051239, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5264259576797485, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7966333627700806, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3094499707221985, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.27855899930000305, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.30525657534599304, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.29247912764549255, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4773670732975006, "metrics/hit_grouping/calorimeter/purity": 0.43131861090660095, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8748413324356079, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06262697279453278, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1523076891899109, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.060970693826675415, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02029372937977314, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05384615436196327, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7944534420967102, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.11784777045249939, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10490307956933975, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.11545994877815247, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1019626185297966, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38198402523994446, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7770135998725891, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3896186947822571, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4239572286605835, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3852338492870331, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3755514323711395, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5329879522323608, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.735519528388977, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44216975569725037, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4571615159511566, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4437197744846344, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.584429144859314, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7986782789230347, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3724060654640198, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.506574809551239, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4714721143245697, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4245653450489044, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6752592325210571, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43971017003059387, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39405861496925354, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3917933702468872, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6699220538139343, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7046998739242554, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4348195791244507, "metrics/hit_grouping/tracker/all/object_completeness": 0.4234796166419983, "metrics/hit_grouping/tracker/all/object_purity": 0.41491633653640747, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6222369074821472, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.012002184055745602, "metrics/hit_grouping/tracker/background_recall": 0.055855002254247665, "metrics/hit_grouping/tracker/completeness": 0.4456128180027008, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5317444801330566, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7151361107826233, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4201366603374481, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47465065121650696, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4702202379703522, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.614669680595398, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.695518434047699, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45021891593933105, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4088954031467438, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4060698449611664, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6322101950645447, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6912881135940552, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4510517716407776, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32538264989852905, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3015486001968384, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6284385323524475, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.728471577167511, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4598310589790344, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44950005412101746, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4390021860599518, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5862688422203064, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7566424012184143, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47603195905685425, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.47484099864959717, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4406995177268982, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5332889556884766, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6645980477333069, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3949894607067108, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3805505335330963, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3790050745010376, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6847342252731323, "metrics/hit_grouping/tracker/purity": 0.4278005063533783, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8689680099487305, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.011294526048004627, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.026259196922183037, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.027678797021508217, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08384013921022415, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6752862334251404, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19699326157569885, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20872299373149872, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20445191860198975, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5484707355499268, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6857277750968933, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5194498300552368, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5019087195396423, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4915001392364502, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7037955522537231, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.42291659116744995, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5667371153831482, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6297670602798462, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.607711672782898, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0628107413649559, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0769069492816925, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.042932510375977, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29857534170150757, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16813550889492035, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8070636987686157, "metrics/particle/class_1/pid_accuracy": 0.949594259262085, "metrics/particle/class_2/efficiency": 0.3529279828071594, "metrics/particle/class_2/pid_accuracy": 0.4818817675113678, "metrics/particle/class_3/efficiency": 0.5927960276603699, "metrics/particle/class_3/pid_accuracy": 0.9506369233131409, "metrics/particle/class_4/efficiency": 0.6891241669654846, "metrics/particle/class_4/pid_accuracy": 0.29050278663635254, "metrics/particle/class_5/efficiency": 0.8759999871253967, "metrics/particle/class_5/pid_accuracy": 0.3652968108654022, "metrics/particle/count/bias_mean": -7.241536617279053, "metrics/particle/count/mae": 7.910807132720947, "metrics/particle/count/prediction_mean": 62.814453125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1153746023774147, "metrics/particle/event/energy_response_mean": 0.907853901386261, "metrics/particle/event/met_abs_error": 10.257365226745605, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09237396717071533, "metrics/particle/event/scalar_pt_response_mean": 0.9278625845909119, "metrics/particle/event/vector_pt_closure": 0.07792980968952179, "metrics/particle/matched/delta_eta_abs_mean": 0.020023036748170853, "metrics/particle/matched/delta_phi_abs_mean": 0.016377193853259087, "metrics/particle/matched/delta_r_mean": 0.028918394818902016, "metrics/particle/matched/energy_relative_abs_mean": 0.1667715311050415, "metrics/particle/matched/pid_accuracy": 0.919748067855835, "metrics/particle/matched/pt_relative_abs_mean": 0.16502645611763, "metrics/particle/matching/duplicate_fraction": 0.07637614756822586, "metrics/particle/matching/efficiency": 0.6758080124855042, "metrics/particle/matching/f1": 0.7126400470733643, "metrics/particle/matching/purity": 0.7537182569503784, "metrics/particle/matching/target_coverage_dr0p05": 0.6057004332542419, "metrics/particle/matching/target_coverage_dr0p10": 0.7299964427947998, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.46981582045555115, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7285746335983276, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027242539450526237, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16108934581279755, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16203106939792633, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004105838946998119, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2025970071554184, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011862069368362427, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9298005700111389, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.16061481833457947, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.5139896869659424, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7250.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32815679907798767, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0713547468185425, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2847543954849243, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1061182022094727, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17032907903194427, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.411697387695312, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01983063481748104, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2963017225265503, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.25748956203460693, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3290669918060303, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9329.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6629480719566345, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -6.839698791503906, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.307805061340332, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.559427261352539, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.75823575258255, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.22342726588249207, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.721915245056152, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.379033088684082, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.231603622436523, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02508990839123726, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17202873528003693, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17367082834243774, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01932840794324875, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21402570605278015, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7537076473236084, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.2552796006202698, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.31849479675293, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1680.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4081505835056305, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7830963730812073, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.896506905555725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7164932489395142, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1756819188594818, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 13.832096099853516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05640314519405365, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.495129346847534, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.32700401544570923, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.028716564178467, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2163.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.556107223033905, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -12.089956283569336, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.287708282470703, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.1306650638580322, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.1285988986492157, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.19318531453609467, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 14.058136940002441, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.4441537857055664, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.269067764282227, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.029989534988999367, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1685221940279007, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1650390625, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -6.251571903703734e-05, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20483210682868958, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001598508097231388, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3243284225463867, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.06236988306045532, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9775779247283936, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15014.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3359385132789612, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7656059265136719, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3438178300857544, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1459028720855713, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.18753430247306824, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.737013816833496, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006836668588221073, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6108716130256653, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.10181403905153275, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.177825450897217, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20039.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7003778219223022, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -5.542156219482422, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.66389274597168, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.332130432128906, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.5821642875671387, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3036653697490692, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.239076614379883, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.830812692642212, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.873631477355957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02881399542093277, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16651763021945953, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1647339165210724, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0025601591914892197, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20481857657432556, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0070998999290168285, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6079497933387756, "metrics/particle/momentum/all/accepted/pz_bias": 0.10565271973609924, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9548474550247192, "metrics/particle/momentum/all/accepted/samples": 23944.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3385898470878601, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.492800235748291, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3642568588256836, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1732739210128784, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18163077533245087, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.611282348632812, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01408138033002615, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9429267048835754, "metrics/particle/momentum/all/assigned/pz_bias": 0.16332121193408966, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9642131328582764, "metrics/particle/momentum/all/assigned/samples": 31531.0, "metrics/particle/momentum/all/efficiency": 0.6764988303184509, "metrics/particle/momentum/all/event/energy_bias": -24.471811294555664, "metrics/particle/momentum/all/event/p3_error_mean": 30.767963409423828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.494959831237793, "metrics/particle/momentum/all/event/px_bias": 2.468998908996582, "metrics/particle/momentum/all/event/py_bias": 0.11294719576835632, "metrics/particle/momentum/all/event/pz_abs_mean": 24.639984130859375, "metrics/particle/momentum/all/event/pz_bias": 9.653999328613281, "metrics/particle/momentum/all/event/pz_residual_rms": 34.696773529052734, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0307132788002491, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16133970022201538, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1587630659341812, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0023435016628354788, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19768908619880676, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007711129263043404, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6374049782752991, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.14307451248168945, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.953920841217041, "metrics/particle/momentum/class_1/accepted/samples": 13487.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1777288317680359, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.45331454277038574, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.49471405148506165, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.31408458948135376, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1028066873550415, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.242076873779297, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01116367056965828, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8108632564544678, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.18493080139160156, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.467681884765625, "metrics/particle/momentum/class_1/assigned/samples": 15855.0, "metrics/particle/momentum/class_1/efficiency": 0.8082339763641357, "metrics/particle/momentum/class_1/event/energy_bias": -1.2793997526168823, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.313173294067383, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.266852378845215, "metrics/particle/momentum/class_1/event/px_bias": 3.00632643699646, "metrics/particle/momentum/class_1/event/py_bias": 1.821405291557312, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.569275856018066, "metrics/particle/momentum/class_1/event/pz_bias": 5.483780860900879, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.345632553100586, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03897469863295555, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21197940409183502, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21108613908290863, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06794155389070511, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25092220306396484, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03326810151338577, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8791252374649048, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0725167989730835, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.401981830596924, "metrics/particle/momentum/class_2/accepted/samples": 1022.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6911119222640991, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7696388959884644, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9130625128746033, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4702494144439697, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33047211170196533, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.5087199211120605, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05418454855680466, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7432167530059814, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.27486056089401245, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.188145637512207, "metrics/particle/momentum/class_2/assigned/samples": 1864.0, "metrics/particle/momentum/class_2/efficiency": 0.34329861402511597, "metrics/particle/momentum/class_2/event/energy_bias": -23.69048309326172, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.41599464416504, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.715982437133789, "metrics/particle/momentum/class_2/event/px_bias": -0.5577377080917358, "metrics/particle/momentum/class_2/event/py_bias": 0.41302037239074707, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.828372955322266, "metrics/particle/momentum/class_2/event/pz_bias": 1.6811408996582031, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.196012496948242, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024700893089175224, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16878952085971832, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16803939640522003, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010766016319394112, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20910272002220154, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002986395265907049, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.38573500514030457, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.040749672800302505, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3655366897583008, "metrics/particle/momentum/class_3/accepted/samples": 9041.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4838494062423706, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.8377530574798584, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.468660593032837, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.3000266551971436, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.25561144948005676, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.138711929321289, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011635762639343739, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8198457956314087, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11518373340368271, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7055976390838623, "metrics/particle/momentum/class_3/assigned/samples": 13321.0, "metrics/particle/momentum/class_3/efficiency": 0.59452885389328, "metrics/particle/momentum/class_3/event/energy_bias": 4.093222141265869, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.161270141601562, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.375205039978027, "metrics/particle/momentum/class_3/event/px_bias": -0.1469167023897171, "metrics/particle/momentum/class_3/event/py_bias": -2.3428542613983154, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.876011848449707, "metrics/particle/momentum/class_3/event/pz_bias": 1.8688181638717651, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.363210678100586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03474285453557968, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1721114218235016, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1680288314819336, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028905218467116356, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20736543834209442, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003921568859368563, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0723987817764282, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.579479455947876, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.4147281646728516, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3409469723701477, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8207868337631226, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6396900415420532, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4947201013565063, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.193452388048172, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.747000694274902, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014880952425301075, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4567949771881104, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.4872492849826813, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.162144660949707, "metrics/particle/momentum/class_4/assigned/samples": 336.0, "metrics/particle/momentum/class_4/efficiency": 0.7005494236946106, "metrics/particle/momentum/class_4/event/energy_bias": -1.958787202835083, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.15492582321167, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.8951950073242188, "metrics/particle/momentum/class_4/event/px_bias": 0.1119287759065628, "metrics/particle/momentum/class_4/event/py_bias": 0.16551698744297028, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4512321949005127, "metrics/particle/momentum/class_4/event/pz_bias": 0.6942604184150696, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.579896926879883, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.026475250720977783, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17663545906543732, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18222682178020477, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.04011855274438858, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22054272890090942, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.028776979073882103, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0051088333129883, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.07054198533296585, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.6582741737365723, "metrics/particle/momentum/class_5/accepted/samples": 139.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06475912779569626, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2199030965566635, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.22441047430038452, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.017653895542025566, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07096774131059647, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2940926253795624, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03870967775583267, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.265651226043701, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.04635452851653099, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1390790939331055, "metrics/particle/momentum/class_5/assigned/samples": 155.0, "metrics/particle/momentum/class_5/efficiency": 0.8742138147354126, "metrics/particle/momentum/class_5/event/energy_bias": -1.6363649368286133, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.009049654006958, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.181729316711426, "metrics/particle/momentum/class_5/event/px_bias": 0.05539806932210922, "metrics/particle/momentum/class_5/event/py_bias": 0.0558587983250618, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6797484159469604, "metrics/particle/momentum/class_5/event/pz_bias": -0.07400090247392654, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.017958641052246, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4801410734653473, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4826219975948334, "metrics/set_slot/angular_recall_all_dr0p10": 0.9170957207679749, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7273386120796204, "metrics/set_slot/soft_count_bias_mean": 0.174530491232872}, "step_train_time": 6.580808877944946, "step_valid_time": 35748.931819200516, "step_total_time": 35755.51262831688, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9697133561970319, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19641385133504785, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2996, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2773, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2731, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9160885365047903, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9255674232309746, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.920803586252698, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07443257676902537, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9022134126197555, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9115487316421896, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9068570479827328, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9160885365047903, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9460080284147836, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19445786958369482, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2216, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2051, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2038, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9067197170645447, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9255415162454874, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9160339437248772, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07445848375451264, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.900972590627763, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9196750902527075, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9102277802590442, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9067197170645447, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9430590940139916, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1891242898658787, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1241, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1140, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1131, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8850931677018633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9186140209508461, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9015421115065243, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08138597904915391, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8781055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.911361804995971, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8944246737841044, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8850931677018633} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_15000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..1e6ee6b64cf2009a7075dda74aeebe3e41ad5518 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7106310403638431, "train": {"Classification_binary": 0.2399289906024933, "Cardinality": 0.19523738324642181, "Classification": 0.11810905486345291, "Regression_pt": 0.2064710259437561, "Regression_eta": 0.002411128953099251, "Regression_sin_phi": 0.0012394717196002603, "Regression_cos_phi": 0.0011991173960268497, "Regression_energy": 0.2046576887369156, "HitGrouping": 0.21893379092216492, "Auxiliary": 0.306104838848114, "Total": 1.5014466047286987, "Total_unweighted": 0.7740164782153443}, "train_group": {}, "valid": {"Classification_binary": 0.23292179902394614, "Cardinality": 0.16964308420817056, "Classification": 0.11997117598851521, "Regression_pt": 0.2000505526860555, "Regression_eta": 0.0023871478624641895, "Regression_sin_phi": 0.0012505863948414724, "Regression_cos_phi": 0.001243773614987731, "Regression_energy": 0.1972583532333374, "HitGrouping": 0.2219433585802714, "Auxiliary": 0.2956248124440511, "Total": 1.459145228068034, "Total_unweighted": 0.7550833888041476, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7710632681846619, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3615897297859192, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.40918105840682983, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.35674434900283813, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3499928116798401, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5359402894973755, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7025938034057617, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.31714746356010437, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4107082486152649, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3079756796360016, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.32695409655570984, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.626906156539917, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7949443459510803, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3705202341079712, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.388028085231781, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.36576586961746216, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3473227620124817, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4905484914779663, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7789760828018188, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3628591299057007, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3968693017959595, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3576792776584625, "metrics/hit_grouping/calorimeter/all/object_purity": 0.34635499119758606, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5173860192298889, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02596762403845787, "metrics/hit_grouping/calorimeter/background_recall": 0.631199061870575, "metrics/hit_grouping/calorimeter/completeness": 0.4340066611766815, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4611499309539795, "metrics/hit_grouping/calorimeter/energy_purity": 0.45587053894996643, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5091814398765564, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.792633056640625, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.36919811367988586, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.32558971643447876, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3656015396118164, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3595840036869049, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.48026275634765625, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7773995399475098, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.38815683126449585, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4432314336299896, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3818841278553009, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3749319911003113, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5255943536758423, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7517117857933044, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3230789601802826, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49943116307258606, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3156866431236267, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.28857359290122986, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5874099135398865, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7722418308258057, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4074481427669525, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4898785352706909, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4033159911632538, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.38381829857826233, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5360323786735535, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7722127437591553, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4991215169429779, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6216640472412109, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4869808852672577, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4566754698753357, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5211930871009827, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7870813608169556, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.29748648405075073, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.26811301708221436, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.29207107424736023, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2919626235961914, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4968528151512146, "metrics/hit_grouping/calorimeter/purity": 0.4276927411556244, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8663932085037231, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05193835869431496, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13692307472229004, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05287179350852966, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.016204599291086197, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05076923221349716, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7938141226768494, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12074501067399979, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1077537089586258, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.11864637583494186, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.10496830940246582, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3648802638053894, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7762595415115356, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38341155648231506, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4190385639667511, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3779241740703583, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3672754168510437, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5361030697822571, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.739417552947998, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.417812317609787, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.444317489862442, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4402685761451721, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5766587257385254, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.797855794429779, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3482894003391266, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47403788566589355, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.43925878405570984, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42063936591148376, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6791749000549316, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4413641095161438, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39387327432632446, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3873184025287628, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6646451950073242, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7082109451293945, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4260816276073456, "metrics/hit_grouping/tracker/all/object_completeness": 0.4165686070919037, "metrics/hit_grouping/tracker/all/object_purity": 0.40842896699905396, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6162856817245483, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015055344440042973, "metrics/hit_grouping/tracker/background_recall": 0.05923118069767952, "metrics/hit_grouping/tracker/completeness": 0.43639981746673584, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5429896712303162, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7172837853431702, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40391799807548523, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46413275599479675, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.46513107419013977, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6047380566596985, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7014819383621216, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45193222165107727, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40384355187416077, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3889275789260864, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6194555759429932, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6952200531959534, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44781553745269775, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32450324296951294, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.30327820777893066, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6385517716407776, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7368864417076111, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4608195424079895, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4473940134048462, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4239964783191681, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5765635967254639, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7552264332771301, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45938748121261597, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46384716033935547, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4450092017650604, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.550599217414856, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6625872254371643, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3749707341194153, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.36809784173965454, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.38171640038490295, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6795832514762878, "metrics/hit_grouping/tracker/purity": 0.42025449872016907, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8866156935691833, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01303214579820633, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0249467846006155, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02628723531961441, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08297132700681686, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6831004023551941, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.189217209815979, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20155072212219238, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1976904720067978, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5463970899581909, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6868279576301575, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5091789364814758, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49426668882369995, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.48435160517692566, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6967121362686157, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.36935627460479736, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5195067524909973, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5711921453475952, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6112417578697205, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11637108772993088, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08043697476387024, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.212190628051758, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28314101696014404, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15270118415355682, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.811139702796936, "metrics/particle/class_1/pid_accuracy": 0.9549944996833801, "metrics/particle/class_2/efficiency": 0.3713259994983673, "metrics/particle/class_2/pid_accuracy": 0.512386679649353, "metrics/particle/class_3/efficiency": 0.5798811316490173, "metrics/particle/class_3/pid_accuracy": 0.946466863155365, "metrics/particle/class_4/efficiency": 0.676612138748169, "metrics/particle/class_4/pid_accuracy": 0.2588904798030853, "metrics/particle/class_5/efficiency": 0.7720000147819519, "metrics/particle/class_5/pid_accuracy": 0.32642486691474915, "metrics/particle/count/bias_mean": -8.259114265441895, "metrics/particle/count/mae": 8.76171875, "metrics/particle/count/prediction_mean": 61.796875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09116214513778687, "metrics/particle/event/energy_response_mean": 0.9841886162757874, "metrics/particle/event/met_abs_error": 10.655760765075684, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07265133410692215, "metrics/particle/event/scalar_pt_response_mean": 0.9949529767036438, "metrics/particle/event/vector_pt_closure": 0.08039844781160355, "metrics/particle/matched/delta_eta_abs_mean": 0.017378825694322586, "metrics/particle/matched/delta_phi_abs_mean": 0.01633792743086815, "metrics/particle/matched/delta_r_mean": 0.026738811284303665, "metrics/particle/matched/energy_relative_abs_mean": 0.16132952272891998, "metrics/particle/matched/pid_accuracy": 0.9214763045310974, "metrics/particle/matched/pt_relative_abs_mean": 0.1633846014738083, "metrics/particle/matching/duplicate_fraction": 0.0776127278804779, "metrics/particle/matching/efficiency": 0.6730479598045349, "metrics/particle/matching/f1": 0.7152069211006165, "metrics/particle/matching/purity": 0.7630004286766052, "metrics/particle/matching/target_coverage_dr0p05": 0.6178744435310364, "metrics/particle/matching/target_coverage_dr0p10": 0.7327843904495239, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.47134917974472046, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7187145948410034, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025044461712241173, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15655267238616943, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1604623645544052, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023706162348389626, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20009703934192657, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011195576749742031, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8936465978622437, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.14578935503959656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3289599418640137, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7235.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28100016713142395, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0946964025497437, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1898118257522583, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0391746759414673, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16657546162605286, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.184525489807129, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022545693442225456, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3360064029693604, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05992970988154411, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5827102661132812, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9137.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6615764498710632, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.9222369194030762, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.85441780090332, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.86591100692749, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.30327919125556946, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6560416221618652, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.275935173034668, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.7232125997543335, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.12415885925293, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025488466024398804, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1644877940416336, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16573549807071686, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02927718125283718, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20699873566627502, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030450670048594475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6563833951950073, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3424081802368164, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.097564220428467, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1642.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.35070469975471497, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9292912483215332, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.7512327432632446, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.5979472398757935, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17440758645534515, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 9.684988975524902, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05876777321100235, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.91471791267395, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.25051993131637573, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.818443298339844, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2110.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5435286164283752, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.003325939178467, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.620194435119629, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.817314624786377, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1303526908159256, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.30135664343833923, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.424261093139648, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6971139907836914, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.245758056640625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027445103973150253, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1623498499393463, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16379143297672272, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018924256786704063, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2043037861585617, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001471768831834197, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3229176700115204, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0030095342081040144, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9927486777305603, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14948.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.31488290429115295, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.18930983543396, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.5478821992874146, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.3775815963745117, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.19066128134727478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.083355903625488, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008480565622448921, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6675681471824646, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004742919001728296, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.631277561187744, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19810.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.697299063205719, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.342475414276123, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.052321434020996, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.010720252990723, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.696591854095459, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.5982048511505127, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.702746391296387, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6915638446807861, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.288407325744629, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02658124640583992, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16073675453662872, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1629144698381424, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.021089905872941017, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2032235562801361, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006421825848519802, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5881338715553284, "metrics/particle/momentum/all/accepted/pz_bias": -0.06975894421339035, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8500362634658813, "metrics/particle/momentum/all/accepted/samples": 23825.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3073483109474182, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7816683053970337, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4563531875610352, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.292993426322937, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18247094750404358, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.63035774230957, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016035031527280807, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.016893744468689, "metrics/particle/momentum/all/assigned/pz_bias": -0.037676919251680374, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.017980575561523, "metrics/particle/momentum/all/assigned/samples": 31057.0, "metrics/particle/momentum/all/efficiency": 0.6731367111206055, "metrics/particle/momentum/all/event/energy_bias": -2.7386131286621094, "metrics/particle/momentum/all/event/p3_error_mean": 27.895580291748047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.98034954071045, "metrics/particle/momentum/all/event/px_bias": 3.869518518447876, "metrics/particle/momentum/all/event/py_bias": -3.55560302734375, "metrics/particle/momentum/all/event/pz_abs_mean": 20.591049194335938, "metrics/particle/momentum/all/event/pz_bias": -4.11189079284668, "metrics/particle/momentum/all/event/pz_residual_rms": 34.89200210571289, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0289570651948452, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1507958471775055, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15140679478645325, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020854869857430458, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19056519865989685, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005897095892578363, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5918397307395935, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.10414610803127289, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7730844020843506, "metrics/particle/momentum/class_1/accepted/samples": 13566.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1633792519569397, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5845146775245667, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5901546478271484, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4398382902145386, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10108234733343124, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.085241794586182, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010760175064206123, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7975508570671082, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.09347184002399445, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.576662540435791, "metrics/particle/momentum/class_1/assigned/samples": 15799.0, "metrics/particle/momentum/class_1/efficiency": 0.8129681944847107, "metrics/particle/momentum/class_1/event/energy_bias": 12.074409484863281, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.20256805419922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.006871223449707, "metrics/particle/momentum/class_1/event/px_bias": 3.1493303775787354, "metrics/particle/momentum/class_1/event/py_bias": 0.09324610233306885, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.120349884033203, "metrics/particle/momentum/class_1/event/pz_bias": -6.936822891235352, "metrics/particle/momentum/class_1/event/pz_residual_rms": 28.095827102661133, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03574952110648155, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2128959447145462, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2126673310995102, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05004596337676048, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2509014308452606, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.034807149320840836, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8638083934783936, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.034468650817871094, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.256180763244629, "metrics/particle/momentum/class_2/accepted/samples": 1063.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5704561471939087, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7015507817268372, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7800624966621399, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.40052831172943115, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2979075014591217, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9463961124420166, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05341409519314766, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.732769727706909, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.055282216519117355, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.844330310821533, "metrics/particle/momentum/class_2/assigned/samples": 1816.0, "metrics/particle/momentum/class_2/efficiency": 0.3570708632469177, "metrics/particle/momentum/class_2/event/energy_bias": -17.25106430053711, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.12577247619629, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.936380386352539, "metrics/particle/momentum/class_2/event/px_bias": 1.8164353370666504, "metrics/particle/momentum/class_2/event/py_bias": -2.740640163421631, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.53049087524414, "metrics/particle/momentum/class_2/event/pz_bias": 1.642797827720642, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.1385440826416, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021717190742492676, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1691669225692749, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17411306500434875, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.028948107734322548, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2149762064218521, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0023833843879401684, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3861241638660431, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.019575143232941628, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.305148720741272, "metrics/particle/momentum/class_3/accepted/samples": 8811.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4481867551803589, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.354517936706543, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.5805823802948, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.4304072856903076, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2654573619365692, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.43065071105957, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01536086481064558, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9974719285964966, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.023878686130046844, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.784687042236328, "metrics/particle/momentum/class_3/assigned/samples": 12955.0, "metrics/particle/momentum/class_3/efficiency": 0.5794042348861694, "metrics/particle/momentum/class_3/event/energy_bias": 3.863508939743042, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.144010543823242, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.234650611877441, "metrics/particle/momentum/class_3/event/px_bias": -1.5638062953948975, "metrics/particle/momentum/class_3/event/py_bias": -1.1741918325424194, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.824374198913574, "metrics/particle/momentum/class_3/event/pz_bias": 1.3558928966522217, "metrics/particle/momentum/class_3/event/pz_residual_rms": 21.044513702392578, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.030173659324645996, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17327038943767548, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17031505703926086, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05694810673594475, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20542845129966736, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.03100775182247162, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.3468128442764282, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.18735086917877197, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.2108776569366455, "metrics/particle/momentum/class_4/accepted/samples": 258.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3321402072906494, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.9424479007720947, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.0228207111358643, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9074461460113525, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19760479032993317, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.532337188720703, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.041916169226169586, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.771554708480835, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.011389917694032192, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.862380027770996, "metrics/particle/momentum/class_4/assigned/samples": 334.0, "metrics/particle/momentum/class_4/efficiency": 0.708791196346283, "metrics/particle/momentum/class_4/event/energy_bias": -1.5386327505111694, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.540374517440796, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.485509157180786, "metrics/particle/momentum/class_4/event/px_bias": 0.07108500599861145, "metrics/particle/momentum/class_4/event/py_bias": -0.021926937624812126, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0233371257781982, "metrics/particle/momentum/class_4/event/pz_bias": 0.1300761103630066, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.07020902633667, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02622024156153202, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17570529878139496, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18374551832675934, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02357727661728859, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2229696810245514, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05511811003088951, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9885362386703491, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6959096193313599, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9385838508605957, "metrics/particle/momentum/class_5/accepted/samples": 127.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07153355330228806, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.32648688554763794, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3166360855102539, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.15108448266983032, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1568627506494522, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5634787082672119, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.11764705926179886, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.2974298000335693, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.649066150188446, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.738919734954834, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.7987421154975891, "metrics/particle/momentum/class_5/event/energy_bias": 0.11316435784101486, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.7220299243927, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.889326810836792, "metrics/particle/momentum/class_5/event/px_bias": 0.39647409319877625, "metrics/particle/momentum/class_5/event/py_bias": 0.2879098355770111, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.8913826942443848, "metrics/particle/momentum/class_5/event/pz_bias": -0.3038346767425537, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.686041831970215, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4642626643180847, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.46816501021385193, "metrics/set_slot/angular_recall_all_dr0p10": 0.9176532626152039, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7365853190422058, "metrics/set_slot/soft_count_bias_mean": -1.7080901861190796}, "step_train_time": 4.416156530380249, "step_valid_time": 38418.720484018326, "step_total_time": 38423.13664102554, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047087794448233, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20628988067218657, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3016, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2808, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2720, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276511397423192, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9310344827586207, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9293397319212312, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06896551724137931, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8985794516022465, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9018567639257294, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9002151249379448, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276511397423192, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9908603383789061, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1934314882666618, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2226, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2090, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2081, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9239610963748894, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9389038634321654, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9313725490196079, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061096136567834684, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9199823165340407, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348607367475292, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9273618538324421, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9239610963748894, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035969565037526, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18215537030853646, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1258, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1152, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1140, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8944099378881988, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9157392686804452, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9049489395129615, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08426073131955485, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8850931677018633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9062003179650239, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8955223880597015, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8944099378881988} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_16000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..51cc5f30d26f733960b1cea3e6336a55934aa176 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.7580064430547659, "train": {"Classification_binary": 0.23732557892799377, "Cardinality": 0.16476526856422424, "Classification": 0.12167104333639145, "Regression_pt": 0.17775991559028625, "Regression_eta": 0.002375067910179496, "Regression_sin_phi": 0.0010873466962948442, "Regression_cos_phi": 0.0011909820605069399, "Regression_energy": 0.1771911233663559, "HitGrouping": 0.22076411545276642, "Auxiliary": 0.2907296419143677, "Total": 1.4158105850219727, "Total_unweighted": 0.7186010578880087}, "train_group": {}, "valid": {"Classification_binary": 0.2237910827000936, "Cardinality": 0.1537237266699473, "Classification": 0.11246366302172343, "Regression_pt": 0.18165475130081177, "Regression_eta": 0.00211793618897597, "Regression_sin_phi": 0.0011229972975949447, "Regression_cos_phi": 0.0011369866163780291, "Regression_energy": 0.18019296725591025, "HitGrouping": 0.2168325980504354, "Auxiliary": 0.27847745021184284, "Total": 1.3627966245015461, "Total_unweighted": 0.702480384381488, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7740758061408997, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.39379996061325073, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42911118268966675, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.38767558336257935, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.38236549496650696, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5196939706802368, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7127000689506531, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.35474371910095215, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4229075014591217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3457617163658142, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.36455491185188293, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6123347878456116, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7945732474327087, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3902532160282135, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.39784112572669983, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3848082423210144, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3705970346927643, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.48698481917381287, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7806336283683777, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3882094919681549, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41015198826789856, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3822286128997803, "metrics/hit_grouping/calorimeter/all/object_purity": 0.37384214997291565, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.508753776550293, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026048343628644943, "metrics/hit_grouping/calorimeter/background_recall": 0.6331114768981934, "metrics/hit_grouping/calorimeter/completeness": 0.44433116912841797, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4697149693965912, "metrics/hit_grouping/calorimeter/energy_purity": 0.46427121758461, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4870833158493042, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7986623048782349, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.40181878209114075, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.336697518825531, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3977259397506714, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3912836015224457, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4609137177467346, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.777635395526886, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4129284620285034, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4505094587802887, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.40589335560798645, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.40263769030570984, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5204997658729553, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.745597243309021, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.33364349603652954, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5235747694969177, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3247649371623993, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.30691641569137573, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5977752208709717, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7657648324966431, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4190845489501953, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49012768268585205, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.41360339522361755, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4118746519088745, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5501090288162231, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7634696364402771, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5035368800163269, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6342229247093201, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.49229997396469116, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.47359123826026917, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.553113579750061, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7988148331642151, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3402591645717621, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2955001890659332, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.33441367745399475, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3201904892921448, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4586179256439209, "metrics/hit_grouping/calorimeter/purity": 0.4379033148288727, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8619487285614014, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08955926448106766, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.17692308127880096, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08713187277317047, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0368567630648613, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.032307691872119904, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7927970290184021, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.15226007997989655, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12599772214889526, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14928749203681946, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.131468266248703, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.34720638394355774, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7782008051872253, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.40814581513404846, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4314664304256439, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.40191617608070374, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.39496496319770813, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5282004475593567, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7447402477264404, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4237895905971527, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.46511393785476685, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.457971453666687, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.569037675857544, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8007308840751648, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34324172139167786, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4952499270439148, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46536949276924133, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4245653450489044, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6938461065292358, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4355359673500061, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4155651926994324, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4077518880367279, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.65007483959198, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7189339399337769, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42405062913894653, "metrics/hit_grouping/tracker/all/object_completeness": 0.4379371404647827, "metrics/hit_grouping/tracker/all/object_purity": 0.4284774661064148, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6054694652557373, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009748152457177639, "metrics/hit_grouping/tracker/background_recall": 0.05520346015691757, "metrics/hit_grouping/tracker/completeness": 0.45847561955451965, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5279756784439087, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7280729413032532, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40792709589004517, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4889448583126068, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.48302021622657776, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6009111404418945, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7128587365150452, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.443746417760849, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4283432066440582, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4230152666568756, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6124119758605957, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.7051011919975281, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43891584873199463, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33490052819252014, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3132035732269287, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6082119941711426, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7474805116653442, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43745505809783936, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46401557326316833, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4555518925189972, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5596693158149719, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7532375454902649, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4354194402694702, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46463972330093384, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4582677483558655, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.550599217414856, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6757133603096008, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40458908677101135, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39926841855049133, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3879683315753937, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6747834086418152, "metrics/hit_grouping/tracker/purity": 0.43914729356765747, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8822396993637085, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.010860121808946133, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02465757541358471, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.029983481392264366, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6804100871086121, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20165888965129852, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21570457518100739, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.21515585482120514, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5500259399414062, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7011299133300781, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5055191516876221, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5193996429443359, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5069161653518677, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6835488080978394, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4178696274757385, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5496726632118225, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6063770055770874, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6221345663070679, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06785769760608673, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09132984280586243, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.977378845214844, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27654358744621277, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14610379934310913, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8342698812484741, "metrics/particle/class_1/pid_accuracy": 0.9600844383239746, "metrics/particle/class_2/efficiency": 0.37155038118362427, "metrics/particle/class_2/pid_accuracy": 0.5398550629615784, "metrics/particle/class_3/efficiency": 0.614077627658844, "metrics/particle/class_3/pid_accuracy": 0.950531005859375, "metrics/particle/class_4/efficiency": 0.6948989629745483, "metrics/particle/class_4/pid_accuracy": 0.2922437787055969, "metrics/particle/class_5/efficiency": 0.871999979019165, "metrics/particle/class_5/pid_accuracy": 0.3600917458534241, "metrics/particle/count/bias_mean": -5.916666507720947, "metrics/particle/count/mae": 6.85546875, "metrics/particle/count/prediction_mean": 64.13932037353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08113198727369308, "metrics/particle/event/energy_response_mean": 0.9635407328605652, "metrics/particle/event/met_abs_error": 9.135173797607422, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06743089109659195, "metrics/particle/event/scalar_pt_response_mean": 0.9791178703308105, "metrics/particle/event/vector_pt_closure": 0.07123643159866333, "metrics/particle/matched/delta_eta_abs_mean": 0.016523241996765137, "metrics/particle/matched/delta_phi_abs_mean": 0.015726806595921516, "metrics/particle/matched/delta_r_mean": 0.02552023157477379, "metrics/particle/matched/energy_relative_abs_mean": 0.1613631695508957, "metrics/particle/matched/pid_accuracy": 0.9280741214752197, "metrics/particle/matched/pt_relative_abs_mean": 0.15932247042655945, "metrics/particle/matching/duplicate_fraction": 0.06888081133365631, "metrics/particle/matching/efficiency": 0.6993848085403442, "metrics/particle/matching/f1": 0.7302206158638, "metrics/particle/matching/purity": 0.7639009952545166, "metrics/particle/matching/target_coverage_dr0p05": 0.6454751491546631, "metrics/particle/matching/target_coverage_dr0p10": 0.7593814730644226, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.49906137585639954, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7474583387374878, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024476194754242897, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16042248904705048, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15986569225788116, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.037188638001680374, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20050136744976044, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010251631028950214, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8490692973136902, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01681177131831646, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2579641342163086, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7511.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3053445518016815, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.13242506980896, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2108914852142334, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0514658689498901, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16909317672252655, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.844542503356934, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019342761486768723, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2499792575836182, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.09989258646965027, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4326012134552, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9616.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6868141889572144, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3387887477874756, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.534634590148926, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.818134307861328, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3268263339996338, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2943946123123169, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.931154251098633, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.797123908996582, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.047024726867676, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023587709292769432, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1680508553981781, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1688614785671234, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04308680072426796, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21107350289821625, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.032767701894044876, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7198280096054077, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08467946946620941, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.332391738891602, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1709.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4359043836593628, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.889729380607605, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9472711086273193, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7911592721939087, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20887166261672974, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 13.860118865966797, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0633074939250946, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.662741184234619, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.32501623034477234, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.125367641448975, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2322.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.565706729888916, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.491390705108643, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.000186920166016, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5002236366271973, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.014011812396347523, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03369053825736046, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.811291694641113, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.7090301513671875, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.849912643432617, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026442594826221466, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15957875549793243, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1570276916027069, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01910778321325779, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1964762955904007, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0012234385358169675, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.285144180059433, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013660328462719917, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8639465570449829, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15530.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.30084043741226196, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7129346132278442, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2568631172180176, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0776042938232422, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.17792926728725433, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.711365699768066, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006619245279580355, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5736365914344788, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03554948419332504, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.393838405609131, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20244.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7244483828544617, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.2222224473953247, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.770332336425781, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.021082878112793, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1642910242080688, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.23232729732990265, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.817087650299072, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8893107771873474, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.680120944976807, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025648709386587143, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16041980683803558, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15870606899261475, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.026250628754496574, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19874261319637299, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006141413934528828, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5553468465805054, "metrics/particle/momentum/all/accepted/pz_bias": 0.009316723793745041, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8197904825210571, "metrics/particle/momentum/all/accepted/samples": 24750.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.31193143129348755, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4800820350646973, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2929413318634033, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1212787628173828, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17752160131931305, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.273611068725586, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01451121736317873, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9264615178108215, "metrics/particle/momentum/all/assigned/pz_bias": 0.07566089183092117, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2846081256866455, "metrics/particle/momentum/all/assigned/samples": 32182.0, "metrics/particle/momentum/all/efficiency": 0.6992710828781128, "metrics/particle/momentum/all/event/energy_bias": -9.052400588989258, "metrics/particle/momentum/all/event/p3_error_mean": 23.60403823852539, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.805055618286133, "metrics/particle/momentum/all/event/px_bias": 1.4771053791046143, "metrics/particle/momentum/all/event/py_bias": 0.49303144216537476, "metrics/particle/momentum/all/event/pz_abs_mean": 17.532184600830078, "metrics/particle/momentum/all/event/pz_bias": 3.3954648971557617, "metrics/particle/momentum/all/event/pz_residual_rms": 27.0282039642334, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027631962671875954, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1506759226322174, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14841926097869873, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0315336212515831, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18682552874088287, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00601245416328311, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5613850355148315, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.036602433770895004, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7974663972854614, "metrics/particle/momentum/class_1/accepted/samples": 13971.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1499861776828766, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.44757044315338135, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4426127076148987, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.302744060754776, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09419386833906174, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.870995283126831, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010279092937707901, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7390251755714417, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.007657876703888178, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.611121416091919, "metrics/particle/momentum/class_1/assigned/samples": 16052.0, "metrics/particle/momentum/class_1/efficiency": 0.8372385501861572, "metrics/particle/momentum/class_1/event/energy_bias": 7.682171821594238, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.211210250854492, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.745076179504395, "metrics/particle/momentum/class_1/event/px_bias": -1.0505543947219849, "metrics/particle/momentum/class_1/event/py_bias": -0.4206520617008209, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.69110107421875, "metrics/particle/momentum/class_1/event/pz_bias": -0.3798558712005615, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.820466995239258, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035989910364151, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20301412045955658, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20449432730674744, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03227631747722626, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24467064440250397, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029918404296040535, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8313827514648438, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.23220854997634888, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.2204909324646, "metrics/particle/momentum/class_2/accepted/samples": 1103.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6701450347900391, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7007600665092468, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7434415221214294, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.32249853014945984, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34987592697143555, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.2952418327331543, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.053101737052202225, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.763108491897583, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.41922855377197266, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.971461772918701, "metrics/particle/momentum/class_2/assigned/samples": 2015.0, "metrics/particle/momentum/class_2/efficiency": 0.37050721049308777, "metrics/particle/momentum/class_2/event/energy_bias": -14.933903694152832, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.563405990600586, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.461804389953613, "metrics/particle/momentum/class_2/event/px_bias": 0.9176706075668335, "metrics/particle/momentum/class_2/event/py_bias": -0.6515220403671265, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.238395690917969, "metrics/particle/momentum/class_2/event/pz_bias": 1.913760781288147, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.589879989624023, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021447110921144485, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16892293095588684, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1679513156414032, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024173520505428314, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20891715586185455, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0028014222625643015, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.35614800453186035, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.04931551218032837, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1961880922317505, "metrics/particle/momentum/class_3/accepted/samples": 9281.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.45225685834884644, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.803520917892456, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.374824047088623, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2009408473968506, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2502386271953583, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.315706253051758, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013216829858720303, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.848541259765625, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.12202770262956619, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.116743326187134, "metrics/particle/momentum/class_3/assigned/samples": 13619.0, "metrics/particle/momentum/class_3/efficiency": 0.6103110313415527, "metrics/particle/momentum/class_3/event/energy_bias": 1.4651877880096436, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.6071195602417, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.020404815673828, "metrics/particle/momentum/class_3/event/px_bias": 1.272323727607727, "metrics/particle/momentum/class_3/event/py_bias": 1.2297641038894653, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.34933853149414, "metrics/particle/momentum/class_3/event/pz_bias": 1.4690996408462524, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.482072830200195, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028398705646395683, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18521349132061005, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17244520783424377, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06585290282964706, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2126905769109726, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.015748031437397003, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1486469507217407, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.36718520522117615, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.576885938644409, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3381882309913635, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.365017890930176, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8066903352737427, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6907726526260376, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22287389636039734, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.052316665649414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020527860149741173, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.350117564201355, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.4579554498195648, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6016852855682373, "metrics/particle/momentum/class_4/assigned/samples": 341.0, "metrics/particle/momentum/class_4/efficiency": 0.6978021860122681, "metrics/particle/momentum/class_4/event/energy_bias": -1.9815645217895508, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.313121795654297, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.309995651245117, "metrics/particle/momentum/class_4/event/px_bias": 0.370846152305603, "metrics/particle/momentum/class_4/event/py_bias": 0.1140260100364685, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9734679460525513, "metrics/particle/momentum/class_4/event/pz_bias": 0.4990096688270569, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.64682149887085, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01984814554452896, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1883309781551361, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18649254739284515, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.026003766804933548, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22828920185565948, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03546099364757538, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0180318355560303, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.46189263463020325, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.959523916244507, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.039020195603370667, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3092724084854126, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3081899583339691, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.15702830255031586, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07741935551166534, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.000610589981079, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05161290243268013, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.375580072402954, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6771665215492249, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.477381229400635, "metrics/particle/momentum/class_5/assigned/samples": 155.0, "metrics/particle/momentum/class_5/efficiency": 0.8867924809455872, "metrics/particle/momentum/class_5/event/energy_bias": -1.2842916250228882, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.787945508956909, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1215057373046875, "metrics/particle/momentum/class_5/event/px_bias": -0.03318069130182266, "metrics/particle/momentum/class_5/event/py_bias": 0.22141540050506592, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4509427547454834, "metrics/particle/momentum/class_5/event/pz_bias": -0.10654924064874649, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.564915180206299, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.44171029329299927, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4437946379184723, "metrics/set_slot/angular_recall_all_dr0p10": 0.929334819316864, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7566213607788086, "metrics/set_slot/soft_count_bias_mean": 0.7371768951416016}, "step_train_time": 4.757055997848511, "step_valid_time": 41077.83383321762, "step_total_time": 41082.590889930725, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999973877479575, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19297920637098465, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3053, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2842, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2799, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9388833828873472, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9308876514903374, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9348684210526316, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06911234850966262, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9246778989098117, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9168031444480839, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9207236842105263, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9388833828873472, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9930041721830817, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17936197076514115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2106, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2090, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9310344827586207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9343389529724934, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9326837909654562, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06566104702750665, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9239610963748894, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.927240461401952, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9255978742249779, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9310344827586207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9882260956235174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1755718418133838, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122670807453416, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9186864738076622, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9154655239579276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08131352619233777, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9021739130434783, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.90852228303362, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.905336969224776, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122670807453416} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_17000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..9249ee302469953bf6ccf682ee1a30e3687007b2 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8053818457456888, "train": {"Classification_binary": 0.2247045934200287, "Cardinality": 0.14083440601825714, "Classification": 0.12003770470619202, "Regression_pt": 0.1730288714170456, "Regression_eta": 0.0020395000465214252, "Regression_sin_phi": 0.0010667460737749934, "Regression_cos_phi": 0.0010915286839008331, "Regression_energy": 0.17117716372013092, "HitGrouping": 0.2117367684841156, "Auxiliary": 0.2786661982536316, "Total": 1.3312430381774902, "Total_unweighted": 0.6931461080675945}, "train_group": {}, "valid": {"Classification_binary": 0.2144729495048523, "Cardinality": 0.1747284730275472, "Classification": 0.10885672767957051, "Regression_pt": 0.1752765973409017, "Regression_eta": 0.002164809498935938, "Regression_sin_phi": 0.0010943282395601273, "Regression_cos_phi": 0.0011218655854463577, "Regression_energy": 0.17347498734792074, "HitGrouping": 0.21148171027501425, "Auxiliary": 0.27906201283137005, "Total": 1.3593966166178386, "Total_unweighted": 0.6764622651971877, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7934972047805786, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.40950900316238403, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3967129588127136, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.40595343708992004, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3984251320362091, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4878624677658081, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7378117442131042, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3589327931404114, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39037615060806274, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3537818491458893, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3756858706474304, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5723483562469482, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.810821533203125, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4044513404369354, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3755810260772705, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4013046622276306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3837827742099762, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4537237882614136, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7986978888511658, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.40199601650238037, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3837082087993622, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3985379636287689, "metrics/hit_grouping/calorimeter/all/object_purity": 0.38776832818984985, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4753495454788208, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025633597746491432, "metrics/hit_grouping/calorimeter/background_recall": 0.629996120929718, "metrics/hit_grouping/calorimeter/completeness": 0.46943315863609314, "metrics/hit_grouping/calorimeter/energy_completeness": 0.493604838848114, "metrics/hit_grouping/calorimeter/energy_purity": 0.4877946972846985, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.46930548548698425, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8160222768783569, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.41037604212760925, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3082711398601532, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.409325510263443, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.40179315209388733, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.42950135469436646, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7972694635391235, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.42838054895401, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.426128089427948, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4236879348754883, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4162043035030365, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.485807865858078, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7635166645050049, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.35676309466362, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49917834997177124, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3494957387447357, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.32844942808151245, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5614966750144958, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7876394391059875, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4411924183368683, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.46085330843925476, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4386839270591736, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.42807191610336304, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5032700300216675, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7777878046035767, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5227991342544556, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6253270506858826, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5143399834632874, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.49441590905189514, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5154369473457336, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8132623434066772, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.344399094581604, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.26985403895378113, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3405602276325226, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.33079245686531067, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.44020354747772217, "metrics/hit_grouping/calorimeter/purity": 0.4624452590942383, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8795573711395264, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07092446088790894, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11538461595773697, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07125640660524368, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.025260673835873604, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8049570322036743, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.15749500691890717, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11288483440876007, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.155114084482193, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14122901856899261, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.34777650237083435, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7966145277023315, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4231152832508087, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.40500393509864807, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4195145070552826, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.40967482328414917, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4918022155761719, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.740684986114502, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4240884780883789, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4656991958618164, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.46707582473754883, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5857740640640259, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7809159159660339, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35053279995918274, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4724343717098236, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46543923020362854, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4503645598888397, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6945591568946838, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4320705533027649, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4184669554233551, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.413932204246521, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6444041728973389, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7164109945297241, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42268088459968567, "metrics/hit_grouping/tracker/all/object_completeness": 0.4379408657550812, "metrics/hit_grouping/tracker/all/object_purity": 0.435067355632782, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6095314621925354, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.007225711364299059, "metrics/hit_grouping/tracker/background_recall": 0.03749333694577217, "metrics/hit_grouping/tracker/completeness": 0.4557840824127197, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5082539916038513, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7215822339057922, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39917996525764465, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4804847538471222, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4916088283061981, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6110250353813171, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.716206431388855, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4458404779434204, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.43065401911735535, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4158048927783966, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6038454174995422, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.7051490545272827, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4502427279949188, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3512416183948517, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.33001959323883057, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6122573018074036, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7455220818519592, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44805896282196045, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46904483437538147, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.45402777194976807, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5610172748565674, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7507721781730652, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44141146540641785, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4678938686847687, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4615580439567566, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5439414381980896, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6724449396133423, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3863263726234436, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39215368032455444, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4057089388370514, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6842659711837769, "metrics/hit_grouping/tracker/purity": 0.4354599714279175, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8820216655731201, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.019113814458251, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03941319137811661, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.040705304592847824, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08514335006475449, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6827418804168701, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20425090193748474, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21966767311096191, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22606267035007477, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5417314767837524, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6977410912513733, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5023906230926514, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5169479250907898, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5124530792236328, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6885071992874146, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4111388325691223, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.553739070892334, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6124345064163208, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5997371077537537, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07458852231502533, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06893239915370941, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.403034210205078, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3029511272907257, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999165534973, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17251132428646088, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8323506116867065, "metrics/particle/class_1/pid_accuracy": 0.964770495891571, "metrics/particle/class_2/efficiency": 0.3530401587486267, "metrics/particle/class_2/pid_accuracy": 0.5640292167663574, "metrics/particle/class_3/efficiency": 0.6188187599182129, "metrics/particle/class_3/pid_accuracy": 0.9581210017204285, "metrics/particle/class_4/efficiency": 0.6785370707511902, "metrics/particle/class_4/pid_accuracy": 0.33191490173339844, "metrics/particle/class_5/efficiency": 0.8820000290870667, "metrics/particle/class_5/pid_accuracy": 0.3900226652622223, "metrics/particle/count/bias_mean": -7.615885257720947, "metrics/particle/count/mae": 8.02734375, "metrics/particle/count/prediction_mean": 62.44010543823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09470134973526001, "metrics/particle/event/energy_response_mean": 0.952775776386261, "metrics/particle/event/met_abs_error": 9.507737159729004, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0748966857790947, "metrics/particle/event/scalar_pt_response_mean": 0.9856538772583008, "metrics/particle/event/vector_pt_closure": 0.07482439279556274, "metrics/particle/matched/delta_eta_abs_mean": 0.018440762534737587, "metrics/particle/matched/delta_phi_abs_mean": 0.015368828549981117, "metrics/particle/matched/delta_r_mean": 0.026838893070816994, "metrics/particle/matched/energy_relative_abs_mean": 0.15807434916496277, "metrics/particle/matched/pid_accuracy": 0.9361478686332703, "metrics/particle/matched/pt_relative_abs_mean": 0.16076602041721344, "metrics/particle/matching/duplicate_fraction": 0.06807565689086914, "metrics/particle/matching/efficiency": 0.6988922357559204, "metrics/particle/matching/f1": 0.739064633846283, "metrics/particle/matching/purity": 0.7841368913650513, "metrics/particle/matching/target_coverage_dr0p05": 0.6385052800178528, "metrics/particle/matching/target_coverage_dr0p10": 0.7564355134963989, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.49251899123191833, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7433600425720215, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02535361610352993, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15840451419353485, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15933626890182495, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03421279788017273, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19962118566036224, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008863819763064384, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8480032682418823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.011645871214568615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.209066390991211, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7446.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2703678011894226, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8833305835723877, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9030619859695435, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7468001246452332, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14651338756084442, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.369338035583496, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018110834062099457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1984812021255493, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0680057555437088, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.346524477005005, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9221.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.680870532989502, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.343186855316162, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.768268585205078, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.933976650238037, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.238030806183815, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.38892659544944763, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.159133911132812, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7662242650985718, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.558669090270996, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02326778881251812, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1712186485528946, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16774706542491913, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.044646065682172775, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20942114293575287, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03001200407743454, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5088962316513062, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.21007505059242249, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6395392417907715, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1666.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.35903993248939514, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7518982887268066, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.62656569480896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.480023741722107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1790352463722229, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.586349487304688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05426716059446335, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.221437692642212, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.3093021512031555, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.423588752746582, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2156.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5514730215072632, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -10.670547485351562, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.831709861755371, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7441601753234863, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17800971865653992, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.059013765305280685, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.641958236694336, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.8689947724342346, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.080001831054688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027764011174440384, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15572483837604523, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16015343368053436, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.04279543086886406, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20066320896148682, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007068046252243221, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2825794517993927, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021167775616049767, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8346495032310486, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15563.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.27978765964508057, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5494089126586914, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2440032958984375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0831807851791382, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16829745471477509, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.00225830078125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005369060207158327, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5299501419067383, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0027770656161010265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8650482892990112, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19929.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7259877920150757, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8983907103538513, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.615950584411621, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.773040771484375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.365730881690979, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.2588086426258087, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.197668552398682, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2070082575082779, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.496339797973633, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026733066886663437, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15757957100868225, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1604195386171341, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.04033046215772629, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20095358788967133, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005146909970790148, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5360015630722046, "metrics/particle/momentum/all/accepted/pz_bias": 0.004347155801951885, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6752122640609741, "metrics/particle/momentum/all/accepted/samples": 24675.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.28247106075286865, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2982962131500244, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1699275970458984, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0114319324493408, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16262058913707733, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.349669456481934, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012489618733525276, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8433523774147034, "metrics/particle/momentum/all/assigned/pz_bias": 0.04309974983334541, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7456815242767334, "metrics/particle/momentum/all/assigned/samples": 31306.0, "metrics/particle/momentum/all/efficiency": 0.6971520781517029, "metrics/particle/momentum/all/event/energy_bias": -13.11534309387207, "metrics/particle/momentum/all/event/p3_error_mean": 26.687726974487305, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.443528175354004, "metrics/particle/momentum/all/event/px_bias": 1.4257519245147705, "metrics/particle/momentum/all/event/py_bias": 0.5887215733528137, "metrics/particle/momentum/all/event/pz_abs_mean": 20.265701293945312, "metrics/particle/momentum/all/event/pz_bias": -0.30977874994277954, "metrics/particle/momentum/all/event/pz_residual_rms": 32.29324722290039, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029432512819767, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14578035473823547, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15197092294692993, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.062301669269800186, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19089052081108093, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005321444012224674, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.553815484046936, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0025454319547861814, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.652890920639038, "metrics/particle/momentum/class_1/accepted/samples": 13906.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13574352860450745, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3813878893852234, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.42388617992401123, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3121234178543091, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09302763640880585, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.295166969299316, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00954773835837841, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7091284394264221, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04679250344634056, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1818549633026123, "metrics/particle/momentum/class_1/assigned/samples": 15920.0, "metrics/particle/momentum/class_1/efficiency": 0.8333433270454407, "metrics/particle/momentum/class_1/event/energy_bias": 7.923508644104004, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.749759674072266, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.229948997497559, "metrics/particle/momentum/class_1/event/px_bias": 1.7794896364212036, "metrics/particle/momentum/class_1/event/py_bias": 1.2719296216964722, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.84811782836914, "metrics/particle/momentum/class_1/event/pz_bias": 1.4275788068771362, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.961944580078125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03389420732855797, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20197941362857819, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20158571004867554, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.034972093999385834, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24112537503242493, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02764067053794861, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7581154108047485, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.19114536046981812, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.02643346786499, "metrics/particle/momentum/class_2/accepted/samples": 1013.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6318784356117249, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6553506255149841, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7906740307807922, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3882961869239807, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3166666626930237, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.302396774291992, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04195402190089226, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.308357000350952, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.15941597521305084, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 4.978609085083008, "metrics/particle/momentum/class_2/assigned/samples": 1740.0, "metrics/particle/momentum/class_2/efficiency": 0.34027543663978577, "metrics/particle/momentum/class_2/event/energy_bias": -23.139358520507812, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.209915161132812, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.357176780700684, "metrics/particle/momentum/class_2/event/px_bias": 1.076445460319519, "metrics/particle/momentum/class_2/event/py_bias": -0.4580889940261841, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.956872940063477, "metrics/particle/momentum/class_2/event/pz_bias": -0.588505744934082, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.891990661621094, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02197662927210331, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16978119313716888, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1679600030183792, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014954710379242897, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2099236696958542, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002026450587436557, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34498587250709534, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021855732426047325, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1168946027755737, "metrics/particle/momentum/class_3/accepted/samples": 9376.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4163878262042999, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4613754749298096, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.1079459190368652, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.923121452331543, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22572557628154755, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.716110229492188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011624373495578766, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7860298752784729, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016072137281298637, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.9096224308013916, "metrics/particle/momentum/class_3/assigned/samples": 13162.0, "metrics/particle/momentum/class_3/efficiency": 0.6165581345558167, "metrics/particle/momentum/class_3/event/energy_bias": 3.948878765106201, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.63766860961914, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.519805908203125, "metrics/particle/momentum/class_3/event/px_bias": -1.701744794845581, "metrics/particle/momentum/class_3/event/py_bias": -0.2701532542705536, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.74345588684082, "metrics/particle/momentum/class_3/event/pz_bias": -1.832844853401184, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.103389739990234, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029614485800266266, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16694651544094086, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17254778742790222, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07709740847349167, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21409808099269867, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01224489789456129, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1134395599365234, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3860121965408325, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.65456485748291, "metrics/particle/momentum/class_4/accepted/samples": 245.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27394071221351624, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.025402307510376, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.180645227432251, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1003031730651855, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21212121844291687, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.301989555358887, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.021212121471762657, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2858723402023315, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2784528434276581, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0594592094421387, "metrics/particle/momentum/class_4/assigned/samples": 330.0, "metrics/particle/momentum/class_4/efficiency": 0.6730769276618958, "metrics/particle/momentum/class_4/event/energy_bias": -0.709485650062561, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.657047748565674, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.444077491760254, "metrics/particle/momentum/class_4/event/px_bias": 0.08861812949180603, "metrics/particle/momentum/class_4/event/py_bias": -0.10895553976297379, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.236712694168091, "metrics/particle/momentum/class_4/event/pz_bias": 0.41480404138565063, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.236505508422852, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0200494397431612, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17539671063423157, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17608049511909485, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03784878924489021, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.217771977186203, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02222222276031971, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7490977048873901, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0845528393983841, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.230623245239258, "metrics/particle/momentum/class_5/accepted/samples": 135.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07556146383285522, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.24312710762023926, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2422989159822464, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.09104782342910767, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11688311398029327, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3497372269630432, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03896103799343109, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.117283582687378, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.15278756618499756, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.0898261070251465, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.849056601524353, "metrics/particle/momentum/class_5/event/energy_bias": -1.1388851404190063, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.057194948196411, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.251920461654663, "metrics/particle/momentum/class_5/event/px_bias": 0.18294352293014526, "metrics/particle/momentum/class_5/event/py_bias": 0.1539897471666336, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6608844995498657, "metrics/particle/momentum/class_5/event/pz_bias": 0.2691890001296997, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.586320877075195, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4408571720123291, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4433063268661499, "metrics/set_slot/angular_recall_all_dr0p10": 0.9323736429214478, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7541215419769287, "metrics/set_slot/soft_count_bias_mean": -2.4663822650909424}, "step_train_time": 4.65131950378418, "step_valid_time": 43682.97540974617, "step_total_time": 43687.62672972679, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0204915664357164, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1888215887312783, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2851, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2815, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9418566237198547, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412347309343018, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941545574636724, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05876526906569825, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299636603898249, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9293496203367448, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9296565389696169, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9418566237198547, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9948898064201843, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1816270037816775, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2100, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2084, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9283819628647215, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9349955476402493, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9316770186335404, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06500445235975066, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9213085764809903, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9278717720391808, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9245785270629991, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9283819628647215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045405161715697, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17594952803265862, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1159, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8998447204968945, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9183835182250396, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9090196078431373, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08161648177496038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8920807453416149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.910459587955626, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9011764705882352, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8998447204968945} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_18000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..ac21c090e6f8ac2eded7064dc887054e6dd9285b --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.8527572484366117, "train": {"Classification_binary": 0.211282417178154, "Cardinality": 0.17583690583705902, "Classification": 0.11133338510990143, "Regression_pt": 0.17238961160182953, "Regression_eta": 0.0021965480409562588, "Regression_sin_phi": 0.0010200702818110585, "Regression_cos_phi": 0.0009034412796609104, "Regression_energy": 0.16999492049217224, "HitGrouping": 0.21175538003444672, "Auxiliary": 0.28127366304397583, "Total": 1.3367620706558228, "Total_unweighted": 0.6691203939844854}, "train_group": {}, "valid": {"Classification_binary": 0.21139399210611978, "Cardinality": 0.13856492439905801, "Classification": 0.10158200065294902, "Regression_pt": 0.17029027144114176, "Regression_eta": 0.0021195118315517902, "Regression_sin_phi": 0.0010853769878546398, "Regression_cos_phi": 0.001079824132223924, "Regression_energy": 0.16831469535827637, "HitGrouping": 0.21014430125554404, "Auxiliary": 0.2665925820668538, "Total": 1.2915964126586914, "Total_unweighted": 0.6558656725101173, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7972368001937866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.42183995246887207, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4005856215953827, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.41971296072006226, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4066074788570404, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.470104843378067, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7413851618766785, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3707493245601654, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3727549910545349, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.368374764919281, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3808329105377197, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.551677405834198, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8124458193778992, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.41375595331192017, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.37397995591163635, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4106212854385376, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3910799026489258, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.44065696001052856, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8011777400970459, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4124997854232788, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.38243159651756287, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4097575545310974, "metrics/hit_grouping/calorimeter/all/object_purity": 0.39515748620033264, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.46009117364883423, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025777706876397133, "metrics/hit_grouping/calorimeter/background_recall": 0.6315198540687561, "metrics/hit_grouping/calorimeter/completeness": 0.47458934783935547, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4961232841014862, "metrics/hit_grouping/calorimeter/energy_purity": 0.49031001329421997, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4481859505176544, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.81966632604599, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.42711392045021057, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3103015720844269, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.42654553055763245, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4150215685367584, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.412302166223526, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8015815019607544, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4387259781360626, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4165453612804413, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.43440043926239014, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4215943217277527, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.46166908740997314, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7616228461265564, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3542363941669464, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49955758452415466, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3485427498817444, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.32556217908859253, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5596005320549011, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7887563705444336, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4484025239944458, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4538150131702423, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4463084638118744, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.43592342734336853, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.49236997961997986, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.772826075553894, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5190576314926147, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6043956279754639, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5120116472244263, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5038664937019348, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.523809552192688, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8181594610214233, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.36026662588119507, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.27728673815727234, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3573783040046692, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.337420791387558, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4172358512878418, "metrics/hit_grouping/calorimeter/purity": 0.46759384870529175, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8492996692657471, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08053836226463318, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1538461595773697, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08011721819639206, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03631727769970894, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04923076927661896, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.79290771484375, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1627083420753479, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11231470853090286, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16105587780475616, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.15264078974723816, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3586089015007019, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8006276488304138, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4339422285556793, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.402839720249176, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4310878813266754, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.41675445437431335, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4746851921081543, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7337893843650818, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.417214572429657, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.46764642000198364, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48292428255081177, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5902570486068726, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7992619276046753, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32473358511924744, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.49853283166885376, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4871678054332733, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4335389733314514, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6736015677452087, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4400252103805542, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.41321665048599243, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4349844753742218, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6794518232345581, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7032080292701721, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42310598492622375, "metrics/hit_grouping/tracker/all/object_completeness": 0.4376055896282196, "metrics/hit_grouping/tracker/all/object_purity": 0.45453187823295593, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6305497884750366, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006051418837159872, "metrics/hit_grouping/tracker/background_recall": 0.03494639694690704, "metrics/hit_grouping/tracker/completeness": 0.4573758542537689, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4949212372303009, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7136113047599792, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39972665905952454, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48833712935447693, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5122758150100708, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6220501065254211, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6965255737304688, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.446601927280426, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4251243472099304, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44226574897766113, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6381115317344666, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6872145533561707, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45004045963287354, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33824992179870605, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3393809199333191, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6413834691047668, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7276591062545776, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4355679452419281, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46435603499412537, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.479734867811203, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5921099781990051, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7278328537940979, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.42809587717056274, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4454004764556885, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.47867491841316223, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5985352993011475, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6670246720314026, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40599390864372253, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4013861119747162, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4174536168575287, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6862561702728271, "metrics/hit_grouping/tracker/purity": 0.4364645183086395, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8863056898117065, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015638574957847595, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.029612891376018524, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.035608597099781036, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.09252823889255524, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6786885857582092, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20943494141101837, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.23071253299713135, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.23758624494075775, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5681700110435486, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6811200380325317, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5028038620948792, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5166029930114746, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5361592173576355, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7107608914375305, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4176211953163147, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5625137686729431, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6217933893203735, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6259908080101013, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06810612231492996, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0951860174536705, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.309059143066406, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29789257049560547, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16745273768901825, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8399485349655151, "metrics/particle/class_1/pid_accuracy": 0.9601187109947205, "metrics/particle/class_2/efficiency": 0.3846758008003235, "metrics/particle/class_2/pid_accuracy": 0.5861767530441284, "metrics/particle/class_3/efficiency": 0.6633342504501343, "metrics/particle/class_3/pid_accuracy": 0.9582147598266602, "metrics/particle/class_4/efficiency": 0.6968238949775696, "metrics/particle/class_4/pid_accuracy": 0.3176795542240143, "metrics/particle/class_5/efficiency": 0.8960000276565552, "metrics/particle/class_5/pid_accuracy": 0.3616071343421936, "metrics/particle/count/bias_mean": -4.220052242279053, "metrics/particle/count/mae": 5.571614742279053, "metrics/particle/count/prediction_mean": 65.8359375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09059115499258041, "metrics/particle/event/energy_response_mean": 0.9913581013679504, "metrics/particle/event/met_abs_error": 9.115373611450195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07730308920145035, "metrics/particle/event/scalar_pt_response_mean": 1.0236386060714722, "metrics/particle/event/vector_pt_closure": 0.07245241105556488, "metrics/particle/matched/delta_eta_abs_mean": 0.0183444581925869, "metrics/particle/matched/delta_phi_abs_mean": 0.015370231121778488, "metrics/particle/matched/delta_r_mean": 0.026738235726952553, "metrics/particle/matched/energy_relative_abs_mean": 0.16052262485027313, "metrics/particle/matched/pid_accuracy": 0.9335145950317383, "metrics/particle/matched/pt_relative_abs_mean": 0.16035057604312897, "metrics/particle/matching/duplicate_fraction": 0.07653969526290894, "metrics/particle/matching/efficiency": 0.724606454372406, "metrics/particle/matching/f1": 0.7471086978912354, "metrics/particle/matching/purity": 0.7710533738136292, "metrics/particle/matching/target_coverage_dr0p05": 0.6582532525062561, "metrics/particle/matching/target_coverage_dr0p10": 0.7814434170722961, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5117837190628052, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7719829678535461, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024639343842864037, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16077513992786407, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16257771849632263, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0331709086894989, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2021760195493698, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010255744680762291, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8491702675819397, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.12810823321342468, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.123854398727417, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7703.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32571518421173096, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5949522256851196, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.9554976224899292, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7939335107803345, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17029021680355072, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 29.42292594909668, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.023864900693297386, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3321819305419922, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.2531585991382599, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.646048069000244, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9889.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7043708562850952, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.761314034461975, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.83261775970459, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.412103652954102, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4342564642429352, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.04039512574672699, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.129195213317871, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.349148273468018, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.9783992767334, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023108171299099922, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16183243691921234, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1640215367078781, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04433068633079529, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2049814611673355, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03442240506410599, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.679438829421997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.173197403550148, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.411791801452637, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1714.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.49846351146698, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9711238145828247, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3454177379608154, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.194739818572998, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21144810318946838, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.107908248901367, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06407517939805984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6321418285369873, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.463738352060318, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.506810188293457, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2341.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5673617720603943, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.2223334312438965, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.478312492370605, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.627228021621704, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.16807399690151215, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15558214485645294, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.286873817443848, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 3.012136220932007, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.48225212097168, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02764766477048397, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16017790138721466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15903478860855103, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015935415402054787, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19911883771419525, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010474430164322257, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.289917528629303, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.07999956607818604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8555502891540527, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16230.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2757365107536316, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.618272304534912, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.236482858657837, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0611381530761719, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16431334614753723, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.406912803649902, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006818181835114956, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5709657669067383, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.11388897150754929, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3357105255126953, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20680.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7571021914482117, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.336611747741699, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.350570678710938, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.233963966369629, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.6332801580429077, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3994271755218506, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.692254543304443, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.144315004348755, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.101366996765137, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02644074708223343, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16046784818172455, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16043215990066528, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023009713739156723, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20043732225894928, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006043592002242804, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5507497787475586, "metrics/particle/momentum/all/accepted/pz_bias": 0.10067731887102127, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.766006350517273, "metrics/particle/momentum/all/accepted/samples": 25647.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3065977394580841, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5652309656143188, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.531419038772583, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3619699478149414, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1694621741771698, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.22018051147461, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01601336896419525, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9463191628456116, "metrics/particle/momentum/all/assigned/pz_bias": 0.18062354624271393, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2302331924438477, "metrics/particle/momentum/all/assigned/samples": 32910.0, "metrics/particle/momentum/all/efficiency": 0.7246143221855164, "metrics/particle/momentum/all/event/energy_bias": -0.12440752983093262, "metrics/particle/momentum/all/event/p3_error_mean": 25.855899810791016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.627808570861816, "metrics/particle/momentum/all/event/px_bias": 1.8994624614715576, "metrics/particle/momentum/all/event/py_bias": -0.5146141648292542, "metrics/particle/momentum/all/event/pz_abs_mean": 20.382944107055664, "metrics/particle/momentum/all/event/pz_bias": 10.505599975585938, "metrics/particle/momentum/all/event/pz_residual_rms": 31.150304794311523, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028821485117077827, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15053397417068481, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15100784599781036, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.054453615099191666, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19029493629932404, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005555159877985716, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5596599578857422, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.1670752614736557, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5461938381195068, "metrics/particle/momentum/class_1/accepted/samples": 14041.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.15241537988185883, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.46447688341140747, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.47185251116752625, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.35236307978630066, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09850120544433594, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.679450511932373, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011225559748709202, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7712198495864868, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.2338857501745224, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5363311767578125, "metrics/particle/momentum/class_1/assigned/samples": 16213.0, "metrics/particle/momentum/class_1/efficiency": 0.8414334654808044, "metrics/particle/momentum/class_1/event/energy_bias": 18.535308837890625, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.638648986816406, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.335007667541504, "metrics/particle/momentum/class_1/event/px_bias": 1.7575678825378418, "metrics/particle/momentum/class_1/event/py_bias": -0.8268312215805054, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.17174530029297, "metrics/particle/momentum/class_1/event/pz_bias": 10.351033210754395, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.138248443603516, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033784978091716766, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19802263379096985, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2027112990617752, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02026577666401863, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24196983873844147, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03583915904164314, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8471890687942505, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04587149620056152, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.478524684906006, "metrics/particle/momentum/class_2/accepted/samples": 1144.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7443610429763794, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.90765780210495, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.200463891029358, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8094518184661865, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3361542224884033, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.11851692199707, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06158909201622009, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7500672340393066, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.3647456467151642, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.217296600341797, "metrics/particle/momentum/class_2/assigned/samples": 2127.0, "metrics/particle/momentum/class_2/efficiency": 0.3842794895172119, "metrics/particle/momentum/class_2/event/energy_bias": -16.637714385986328, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.860383987426758, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.139623641967773, "metrics/particle/momentum/class_2/event/px_bias": 0.33457881212234497, "metrics/particle/momentum/class_2/event/py_bias": -1.8796532154083252, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.61028003692627, "metrics/particle/momentum/class_2/event/pz_bias": 1.8704705238342285, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.492826461791992, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02221687324345112, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16966989636421204, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16826383769512177, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.01730545237660408, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2082533985376358, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022846926003694534, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3502645194530487, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004289764445275068, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3570636510849, "metrics/particle/momentum/class_3/accepted/samples": 10067.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.41790902614593506, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.8883204460144043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.779740333557129, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5846054553985596, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22559544444084167, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.329381942749023, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013579808175563812, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8408637046813965, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07772357016801834, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.2035253047943115, "metrics/particle/momentum/class_3/assigned/samples": 14065.0, "metrics/particle/momentum/class_3/efficiency": 0.6619977355003357, "metrics/particle/momentum/class_3/event/energy_bias": 0.8038820028305054, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.66868782043457, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.739303112030029, "metrics/particle/momentum/class_3/event/px_bias": -0.7721641063690186, "metrics/particle/momentum/class_3/event/py_bias": 2.2547109127044678, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.478015899658203, "metrics/particle/momentum/class_3/event/pz_bias": -2.5034360885620117, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.01171875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.030708078294992447, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.168204203248024, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17015261948108673, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07244884967803955, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21205049753189087, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.023622047156095505, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2670725584030151, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.698619544506073, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.033876657485962, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.43102166056632996, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.991795063018799, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.0476596355438232, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.935328960418701, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2234957069158554, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.626169204711914, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.034383952617645264, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6507898569107056, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.6149230599403381, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.037725448608398, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.6978021860122681, "metrics/particle/momentum/class_4/event/energy_bias": -1.3646283149719238, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.507037401199341, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2552871704101562, "metrics/particle/momentum/class_4/event/px_bias": 0.2906988859176636, "metrics/particle/momentum/class_4/event/py_bias": -0.15887293219566345, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.266306161880493, "metrics/particle/momentum/class_4/event/pz_bias": 0.6868280172348022, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.600320816040039, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02366146631538868, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17406105995178223, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17921686172485352, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.032219041138887405, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22143758833408356, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04964539036154747, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.168508768081665, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.35053980350494385, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.291118144989014, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04774007573723793, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.21306221187114716, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.22302576899528503, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07022016495466232, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.29907408356666565, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07051282376050949, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.4827544689178467, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4405609965324402, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.856594085693359, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8867924809455872, "metrics/particle/momentum/class_5/event/energy_bias": -1.4612560272216797, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.870041608810425, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.087092638015747, "metrics/particle/momentum/class_5/event/px_bias": 0.28878092765808105, "metrics/particle/momentum/class_5/event/py_bias": 0.09603212773799896, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6167981624603271, "metrics/particle/momentum/class_5/event/pz_bias": 0.10070449858903885, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.190065383911133, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.43960094451904297, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.44161245226860046, "metrics/set_slot/angular_recall_all_dr0p10": 0.9328569173812866, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7714067697525024, "metrics/set_slot/soft_count_bias_mean": -0.017529388889670372}, "step_train_time": 14.295937776565552, "step_valid_time": 46328.05532979965, "step_total_time": 46342.351267814636, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0385328886961571, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19733006330918612, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2824, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2776, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9329369012223323, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.930171277997365, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9315520369454066, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06982872200263504, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9170796167822927, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9143610013175231, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9157182912749464, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9329369012223323, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0225353699383994, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19643788598543133, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2100, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2081, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9283819628647215, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9320905459387483, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9302325581395349, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06790945406125166, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9199823165340407, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9236573457612073, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9218161683277962, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9283819628647215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0194095769232248, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18045689087517747, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1169, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1159, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.907608695652174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9219242902208202, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9147104851330203, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07807570977917981, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8998447204968945, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9140378548895899, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.906885758998435, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.907608695652174} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_19000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1453b84c9065476f5422466ac2c8149f085a13 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.9001326511275346, "train": {"Classification_binary": 0.2039867788553238, "Cardinality": 0.17303766310214996, "Classification": 0.12151188403367996, "Regression_pt": 0.18919140100479126, "Regression_eta": 0.002181901363655925, "Regression_sin_phi": 0.0011899364180862904, "Regression_cos_phi": 0.0010781397577375174, "Regression_energy": 0.18796378374099731, "HitGrouping": 0.2171870321035385, "Auxiliary": 0.2890188992023468, "Total": 1.3888485431671143, "Total_unweighted": 0.7071038251742721}, "train_group": {}, "valid": {"Classification_binary": 0.20958890517552695, "Cardinality": 0.1690056324005127, "Classification": 0.10560431083043416, "Regression_pt": 0.17021648089090982, "Regression_eta": 0.0018982072360813618, "Regression_sin_phi": 0.0010537123307585716, "Regression_cos_phi": 0.0010684917991360028, "Regression_energy": 0.16894819339116415, "HitGrouping": 0.21017096439997354, "Auxiliary": 0.272757609685262, "Total": 1.31821870803833, "Total_unweighted": 0.658378301654011, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.789376974105835, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.40373867750167847, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.39727967977523804, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4014795422554016, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4007473587989807, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4857844412326813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7363875508308411, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.34210440516471863, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3791934847831726, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.33949247002601624, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3572872579097748, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5692985653877258, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8118545413017273, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4188261330127716, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.38002273440361023, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.41646289825439453, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.40026038885116577, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.44814586639404297, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7978523373603821, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4070894420146942, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.38550153374671936, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4047373831272125, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3965626060962677, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4711246192455292, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025814352557063103, "metrics/hit_grouping/calorimeter/background_recall": 0.6310880184173584, "metrics/hit_grouping/calorimeter/completeness": 0.4691011607646942, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4873788058757782, "metrics/hit_grouping/calorimeter/energy_purity": 0.4816861152648926, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.455549955368042, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8141626119613647, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4165239632129669, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.30761420726776123, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.41610220074653625, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.41069796681404114, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.42932218313217163, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8008254170417786, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4371948838233948, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.42770498991012573, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4341958165168762, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.42453840374946594, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.46846190094947815, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7602304816246033, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3557469844818115, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5063835382461548, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.34998348355293274, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.33748918771743774, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5623815059661865, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7868257164955139, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.44776856899261475, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4618498980998993, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.44626590609550476, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.441543847322464, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.49996885657310486, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7818494439125061, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5174779891967773, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6059654355049133, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5116284489631653, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4891330301761627, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4908424913883209, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8117545247077942, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.34923121333122253, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2752109169960022, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3464135527610779, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3363593518733978, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4375920593738556, "metrics/hit_grouping/calorimeter/purity": 0.4622039496898651, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8668418526649475, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0637618899345398, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16307692229747772, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0659780204296112, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.029338639229536057, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8064189553260803, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.16568523645401, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11858608573675156, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16352832317352295, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1506376713514328, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3369441330432892, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.795889139175415, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.42829179763793945, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4055941700935364, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.425831139087677, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4185342490673065, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.487900048494339, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7330475449562073, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41452479362487793, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.46897944808006287, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48343896865844727, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5921996235847473, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7948071360588074, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32922041416168213, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4759303629398346, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45499271154403687, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4391475021839142, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6837161779403687, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43892258405685425, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4251035451889038, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4352131485939026, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6577144265174866, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7086642980575562, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4219724237918854, "metrics/hit_grouping/tracker/all/object_completeness": 0.4432520866394043, "metrics/hit_grouping/tracker/all/object_purity": 0.4521220028400421, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6186000108718872, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0043250564485788345, "metrics/hit_grouping/tracker/background_recall": 0.026831723749637604, "metrics/hit_grouping/tracker/completeness": 0.4628714919090271, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4849402904510498, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7195713520050049, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3971754014492035, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.49055391550064087, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.506155788898468, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6127562522888184, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.70731520652771, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44964781403541565, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4351559579372406, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4337838888168335, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6116504669189453, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6858855485916138, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44761326909065247, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34685075283050537, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.35165858268737793, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6389563083648682, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7370113730430603, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4435657858848572, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4761669337749481, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.476531445980072, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5708123445510864, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7310681939125061, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4094540476799011, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4596134126186371, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4956902265548706, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5739014744758606, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6677960157394409, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3960430920124054, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39749574661254883, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41266191005706787, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6887145638465881, "metrics/hit_grouping/tracker/purity": 0.44094303250312805, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8768776655197144, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.017810599878430367, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.040825314819812775, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.047761909663677216, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.09209383279085159, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6759923100471497, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2089165300130844, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.23648707568645477, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.24627019464969635, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5660964250564575, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.689527153968811, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5011510252952576, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5214786529541016, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5305073261260986, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6961218118667603, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.41951984167099, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5775347352027893, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6387060284614563, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5991613268852234, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06620752066373825, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06835652887821198, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.866788864135742, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2772926092147827, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1468527913093567, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8702611923217773, "metrics/particle/class_1/pid_accuracy": 0.9616897702217102, "metrics/particle/class_2/efficiency": 0.4160870611667633, "metrics/particle/class_2/pid_accuracy": 0.6462658643722534, "metrics/particle/class_3/efficiency": 0.6726235151290894, "metrics/particle/class_3/pid_accuracy": 0.9510732889175415, "metrics/particle/class_4/efficiency": 0.7228103876113892, "metrics/particle/class_4/pid_accuracy": 0.312916100025177, "metrics/particle/class_5/efficiency": 0.9100000262260437, "metrics/particle/class_5/pid_accuracy": 0.3978022038936615, "metrics/particle/count/bias_mean": -2.666015625, "metrics/particle/count/mae": 5.242838382720947, "metrics/particle/count/prediction_mean": 67.38997650146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09186054021120071, "metrics/particle/event/energy_response_mean": 0.9512638449668884, "metrics/particle/event/met_abs_error": 8.646957397460938, "metrics/particle/event/scalar_pt_relative_abs_error": 0.076934315264225, "metrics/particle/event/scalar_pt_response_mean": 0.9595690369606018, "metrics/particle/event/vector_pt_closure": 0.07139987498521805, "metrics/particle/matched/delta_eta_abs_mean": 0.016918044537305832, "metrics/particle/matched/delta_phi_abs_mean": 0.016119027510285378, "metrics/particle/matched/delta_r_mean": 0.02614695206284523, "metrics/particle/matched/energy_relative_abs_mean": 0.15623655915260315, "metrics/particle/matched/pid_accuracy": 0.9336955547332764, "metrics/particle/matched/pt_relative_abs_mean": 0.15432871878147125, "metrics/particle/matching/duplicate_fraction": 0.07604988664388657, "metrics/particle/matching/efficiency": 0.7457855343818665, "metrics/particle/matching/f1": 0.7602514028549194, "metrics/particle/matching/purity": 0.7752895951271057, "metrics/particle/matching/target_coverage_dr0p05": 0.6765515208244324, "metrics/particle/matching/target_coverage_dr0p10": 0.7949184775352478, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5425440669059753, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7941100001335144, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02547755464911461, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15495625138282776, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15394921600818634, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.018398912623524666, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19237099587917328, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01036793552339077, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.83338463306427, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.027843259274959564, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.263082504272461, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7909.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32480093836784363, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3537791967391968, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.380384087562561, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1771117448806763, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16147026419639587, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.589252471923828, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021018030121922493, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2492583990097046, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04093310236930847, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5064644813537598, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10039.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7232077717781067, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -5.339667320251465, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.621254920959473, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.769301414489746, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.9654704928398132, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.37559184432029724, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.098105430603027, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.924530565738678, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.579804420471191, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024993931874632835, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16385209560394287, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16353856027126312, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.022901201620697975, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20176172256469727, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.034522440284490585, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6860319375991821, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16023577749729156, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.062614917755127, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1738.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5642974376678467, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9013232588768005, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.029371738433838, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8099396228790283, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21232306957244873, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.660605430603027, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06452956050634384, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7489547729492188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.02988436445593834, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.590352535247803, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2402.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5753061771392822, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.118109703063965, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.983924865722656, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7530187368392944, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.037004582583904266, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07160745561122894, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.78116226196289, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5120371580123901, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.10735511779785, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026454584673047066, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15505053102970123, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1527281403541565, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.01181823294609785, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19211445748806, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010169896995648742, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2738507091999054, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010743976570665836, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8538684844970703, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16716.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.26129835844039917, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3930424451828003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0442394018173218, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8428568243980408, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15086185932159424, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.149796485900879, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0052816057577729225, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4987114369869232, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.008370285853743553, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9889174699783325, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20827.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7797732949256897, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.2556479573249817, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.396341323852539, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.613958358764648, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -4.392251491546631, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.056369274854660034, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.108207702636719, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.17906057834625244, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.327287673950195, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026065178215503693, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15560248494148254, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1538071632385254, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.014523114077746868, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19284206628799438, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006031180266290903, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5348120927810669, "metrics/particle/momentum/all/accepted/pz_bias": -0.025729183107614517, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7569494247436523, "metrics/particle/momentum/all/accepted/samples": 26363.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.30233797430992126, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.345691442489624, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.216802954673767, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0135468244552612, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15850065648555756, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.698113441467285, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014308043755590916, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8876685500144958, "metrics/particle/momentum/all/assigned/pz_bias": 0.009269616566598415, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.216597080230713, "metrics/particle/momentum/all/assigned/samples": 33268.0, "metrics/particle/momentum/all/efficiency": 0.7448437809944153, "metrics/particle/momentum/all/event/energy_bias": -14.713425636291504, "metrics/particle/momentum/all/event/p3_error_mean": 25.712345123291016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.197585105895996, "metrics/particle/momentum/all/event/px_bias": -5.394726276397705, "metrics/particle/momentum/all/event/py_bias": 0.3603536784648895, "metrics/particle/momentum/all/event/pz_abs_mean": 19.311412811279297, "metrics/particle/momentum/all/event/pz_bias": -0.7665669918060303, "metrics/particle/momentum/all/event/pz_residual_rms": 30.452608108520508, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02725423499941826, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13972394168376923, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13816608488559723, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.01621514931321144, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1748809665441513, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005764084402471781, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5419886112213135, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0013964833924546838, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7385318279266357, "metrics/particle/momentum/class_1/accepted/samples": 14573.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14344854652881622, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.38653287291526794, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4339616000652313, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2572863698005676, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07463967800140381, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4619674682617188, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00993560254573822, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7034591436386108, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02967894822359085, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3805394172668457, "metrics/particle/momentum/class_1/assigned/samples": 16305.0, "metrics/particle/momentum/class_1/efficiency": 0.8733145594596863, "metrics/particle/momentum/class_1/event/energy_bias": -0.8780523538589478, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.784225463867188, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.466829299926758, "metrics/particle/momentum/class_1/event/px_bias": -3.1221022605895996, "metrics/particle/momentum/class_1/event/py_bias": -2.663376808166504, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.386392593383789, "metrics/particle/momentum/class_1/event/pz_bias": -0.8103229999542236, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.38964080810547, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03295278921723366, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2093496173620224, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21239116787910461, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.056071508675813675, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24839729070663452, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03293918818235397, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8619921207427979, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19451092183589935, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.177245616912842, "metrics/particle/momentum/class_2/accepted/samples": 1184.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7433874607086182, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7905619740486145, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7993040680885315, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.36166733503341675, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33423912525177, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.396132707595825, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05525362491607666, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.744746446609497, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.12053106725215912, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.5293731689453125, "metrics/particle/momentum/class_2/assigned/samples": 2208.0, "metrics/particle/momentum/class_2/efficiency": 0.39771580696105957, "metrics/particle/momentum/class_2/event/energy_bias": -7.575917720794678, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.7779598236084, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.051468849182129, "metrics/particle/momentum/class_2/event/px_bias": -1.1687684059143066, "metrics/particle/momentum/class_2/event/py_bias": 3.368985414505005, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.413811683654785, "metrics/particle/momentum/class_2/event/pz_bias": 0.9298208355903625, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.055280685424805, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023613519966602325, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17149022221565247, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16902613639831543, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.00834281463176012, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20893403887748718, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0028448107186704874, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3355397880077362, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04693225771188736, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1049073934555054, "metrics/particle/momentum/class_3/accepted/samples": 10194.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4183987081050873, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4657862186431885, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.148233652114868, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.949205994606018, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2276776283979416, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.505512237548828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012344812974333763, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7830169796943665, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03349079191684723, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.228452444076538, "metrics/particle/momentum/class_3/assigned/samples": 14257.0, "metrics/particle/momentum/class_3/efficiency": 0.6703491806983948, "metrics/particle/momentum/class_3/event/energy_bias": -3.0014052391052246, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.844179153442383, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.0043792724609375, "metrics/particle/momentum/class_3/event/px_bias": -0.7460036277770996, "metrics/particle/momentum/class_3/event/py_bias": -0.5442919135093689, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.197980880737305, "metrics/particle/momentum/class_3/event/pz_bias": -1.4971275329589844, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.618074417114258, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02766958437860012, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16879598796367645, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15590471029281616, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03757490590214729, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19488908350467682, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011278195306658745, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0341556072235107, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2978977560997009, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.41135835647583, "metrics/particle/momentum/class_4/accepted/samples": 266.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.30620115995407104, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.486349582672119, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.867278814315796, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7473092079162598, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.18421052396297455, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.220858573913574, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.023391813039779663, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3953912258148193, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.24264758825302124, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.102849006652832, "metrics/particle/momentum/class_4/assigned/samples": 342.0, "metrics/particle/momentum/class_4/efficiency": 0.7307692170143127, "metrics/particle/momentum/class_4/event/energy_bias": -1.5491669178009033, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.096987247467041, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2173681259155273, "metrics/particle/momentum/class_4/event/px_bias": -0.12957093119621277, "metrics/particle/momentum/class_4/event/py_bias": 0.039370305836200714, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8125089406967163, "metrics/particle/momentum/class_4/event/pz_bias": 0.6226223707199097, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.655458450317383, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.019780131056904793, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.171303391456604, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17349018156528473, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.035130809992551804, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21160079538822174, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.027397260069847107, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0594067573547363, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.19493131339550018, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.284884452819824, "metrics/particle/momentum/class_5/accepted/samples": 146.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.051417913287878036, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20153118669986725, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20521362125873566, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.027872886508703232, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2805079221725464, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.3074231147766113, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3023940920829773, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.611817836761475, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.9182389974594116, "metrics/particle/momentum/class_5/event/energy_bias": -1.7088826894760132, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.6067233085632324, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9119770526885986, "metrics/particle/momentum/class_5/event/px_bias": -0.22828134894371033, "metrics/particle/momentum/class_5/event/py_bias": 0.15966667234897614, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4394512176513672, "metrics/particle/momentum/class_5/event/pz_bias": -0.011559614911675453, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.69984245300293, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42522814869880676, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42609864473342896, "metrics/set_slot/angular_recall_all_dr0p10": 0.9354590177536011, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7852350473403931, "metrics/set_slot/soft_count_bias_mean": 1.769253134727478}, "step_train_time": 4.85492730140686, "step_valid_time": 48812.82386159897, "step_total_time": 48817.67878937721, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9876851777622098, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18724216005735284, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3005, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2818, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2795, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9309547406673274, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9377703826955075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9343501326259946, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.062229617304492515, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9233564585398084, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.930116472545757, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9267241379310345, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9309547406673274, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9612629547084186, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17673325530941875, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2225, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2092, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2084, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9248452696728559, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402247191011236, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9324715845776688, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059775280898876404, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9213085764809903, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366292134831461, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9289057276576778, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9248452696728559, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9522644726566546, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16905030924158448, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1167, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1155, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.906055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9254559873116575, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9156531973322871, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07454401268834258, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8967391304347826, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.91593973037272, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9062377402903099, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.906055900621118} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_2000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..3d56e06132dc0841a9fb20fad2a8f5dc9ffe89c3 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.09475080538184574, "train": {"Classification_binary": 0.3919795751571655, "Cardinality": 0.2834584414958954, "Classification": 0.32246536016464233, "Regression_pt": 0.4789411425590515, "Regression_eta": 0.006104201078414917, "Regression_sin_phi": 0.0029424927197396755, "Regression_cos_phi": 0.0029009943827986717, "Regression_energy": 0.47587162256240845, "HitGrouping": 0.3167118728160858, "Auxiliary": 0.5343414545059204, "Total": 2.842973232269287, "Total_unweighted": 1.681205388624221}, "train_group": {}, "valid": {"Classification_binary": 0.4069379965464274, "Cardinality": 0.26562758286794025, "Classification": 0.3371705214182536, "Regression_pt": 0.49995946884155273, "Regression_eta": 0.005877617125709851, "Regression_sin_phi": 0.0030344494928916297, "Regression_cos_phi": 0.0030457467461625734, "Regression_energy": 0.4923702875773112, "HitGrouping": 0.31232742468516034, "Auxiliary": 0.5489699840545654, "Total": 2.88793150583903, "Total_unweighted": 1.7483960877483091, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8343110084533691, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.07535062730312347, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.17408141493797302, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.07290292531251907, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.04578129202127457, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4334561228752136, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8355337977409363, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.07104361802339554, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.13385292887687683, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.06888904422521591, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.04292134940624237, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.39918673038482666, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8053066730499268, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.07611135393381119, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.17059187591075897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.07485049217939377, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.05199746787548065, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5124470591545105, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8173513412475586, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.07541199028491974, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.16841945052146912, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.0736890584230423, "metrics/hit_grouping/calorimeter/all/object_purity": 0.04918305575847626, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4768693149089813, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02630363032221794, "metrics/hit_grouping/calorimeter/background_recall": 0.6269794702529907, "metrics/hit_grouping/calorimeter/completeness": 0.1854100525379181, "metrics/hit_grouping/calorimeter/energy_completeness": 0.22661398351192474, "metrics/hit_grouping/calorimeter/energy_purity": 0.22411592304706573, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5722996592521667, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.821826696395874, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.06236695125699043, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.1226634830236435, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.0604318231344223, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.05619152635335922, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4553598165512085, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8208820223808289, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.07823774218559265, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.1768558919429779, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.07638997584581375, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.043367814272642136, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.471615731716156, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8041992783546448, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.10007941722869873, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.25647830963134766, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.09893567860126495, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.0404084250330925, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5278725624084473, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8015781044960022, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.08341769129037857, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.21401432156562805, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.08082196861505508, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.050168123096227646, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.525070071220398, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8332333564758301, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.33831003308296204, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5557299852371216, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3421091139316559, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.18280674517154694, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4652014672756195, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8322761654853821, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.033164139837026596, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.06984063237905502, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.0316728800535202, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.031025150790810585, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4265434443950653, "metrics/hit_grouping/calorimeter/purity": 0.18277575075626373, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8958119750022888, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.019969427958130836, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.05999999865889549, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.020435897633433342, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.001491853967308998, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04461538419127464, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8477310538291931, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.025294724851846695, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.05074116215109825, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.024139635264873505, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.016956837847828865, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2970353364944458, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8139316439628601, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.0794762447476387, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.1774986833333969, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.07767398655414581, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.052053067833185196, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4964257478713989, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6925608515739441, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3203825354576111, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.17249639332294464, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.1256423145532608, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.687836229801178, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7954484820365906, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2770611345767975, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.20146194100379944, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.11893842369318008, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45204710960388184, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6174912452697754, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.33283454179763794, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.16764964163303375, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.1380901038646698, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7561628818511963, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6562058329582214, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.324201762676239, "metrics/hit_grouping/tracker/all/object_completeness": 0.17202907800674438, "metrics/hit_grouping/tracker/all/object_purity": 0.13254277408123016, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7089552283287048, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0030074086971580982, "metrics/hit_grouping/tracker/background_recall": 0.005330806132405996, "metrics/hit_grouping/tracker/completeness": 0.18359734117984772, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.616416335105896, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6407519578933716, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.33230069279670715, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.20082569122314453, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.17836588621139526, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7330296039581299, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6624510288238525, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.30439749360084534, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.145175039768219, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.0967063456773758, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7049305438995361, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6838756203651428, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3272653818130493, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.13663700222969055, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.06889792531728745, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6597896218299866, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6928926706314087, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.34867003560066223, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.1770525425672531, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.12285793572664261, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6719086766242981, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.716177761554718, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41145139932632446, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.23834827542304993, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.13833074271678925, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6151797771453857, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.597866952419281, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.276984304189682, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.15382345020771027, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.14414185285568237, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7737063765525818, "metrics/hit_grouping/tracker/purity": 0.1746682971715927, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9092127680778503, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012163336388766766, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.01297396793961525, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.007206262089312077, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05777584761381149, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7186897397041321, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.11249352246522903, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.06342113018035889, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.0343729704618454, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5235873460769653, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6147114634513855, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.39070892333984375, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.20600877702236176, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.1607521027326584, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.818546712398529, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 0.25, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.25288328528404236, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5830920934677124, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6514120101928711, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5307941436767578, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.33624786138534546, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -1.0651536285877228e-05, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.5891311168670654, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.699999988079071, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.890432357788086, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.103566013276577, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.7875000238418579, "metrics/linear_probe/tracker/embedding_minus_raw/ari": -0.026873812079429626, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.40391770005226135, "metrics/particle/class_1/pid_accuracy": 0.8160576224327087, "metrics/particle/class_2/efficiency": 0.33138883113861084, "metrics/particle/class_2/pid_accuracy": 0.048747461289167404, "metrics/particle/class_3/efficiency": 0.3767189383506775, "metrics/particle/class_3/pid_accuracy": 0.870330274105072, "metrics/particle/class_4/efficiency": 0.4032723903656006, "metrics/particle/class_4/pid_accuracy": 0.01193317398428917, "metrics/particle/class_5/efficiency": 0.3919999897480011, "metrics/particle/class_5/pid_accuracy": 0.015306122601032257, "metrics/particle/count/bias_mean": -10.551432609558105, "metrics/particle/count/mae": 13.117838859558105, "metrics/particle/count/prediction_mean": 59.50455856323242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.20370988547801971, "metrics/particle/event/energy_response_mean": 0.9773540496826172, "metrics/particle/event/met_abs_error": 16.819231033325195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.18895770609378815, "metrics/particle/event/scalar_pt_response_mean": 1.0154948234558105, "metrics/particle/event/vector_pt_closure": 0.13227759301662445, "metrics/particle/matched/delta_eta_abs_mean": 0.024251962080597878, "metrics/particle/matched/delta_phi_abs_mean": 0.023203492164611816, "metrics/particle/matched/delta_r_mean": 0.03744431585073471, "metrics/particle/matched/energy_relative_abs_mean": 0.2112032026052475, "metrics/particle/matched/pid_accuracy": 0.7725489139556885, "metrics/particle/matched/pt_relative_abs_mean": 0.21103322505950928, "metrics/particle/matching/duplicate_fraction": 0.10192671418190002, "metrics/particle/matching/efficiency": 0.38606584072113037, "metrics/particle/matching/f1": 0.4175071120262146, "metrics/particle/matching/purity": 0.45452356338500977, "metrics/particle/matching/target_coverage_dr0p05": 0.3206048011779785, "metrics/particle/matching/target_coverage_dr0p10": 0.479359894990921, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.23501478135585785, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.4688400328159332, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03753320500254631, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.21700340509414673, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.21484984457492828, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.007395537570118904, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.25610581040382385, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.025111883878707886, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.6399903297424316, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03186739236116409, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.6116719245910645, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4022.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8305911421775818, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.3230931758880615, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.325152397155762, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 4.136207580566406, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.4336867034435272, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.453296661376953, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.043580684810876846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.4776129722595215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.229252889752388, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.268864631652832, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8490.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.36777615547180176, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 6.194887161254883, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 21.83780288696289, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 9.673961639404297, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.338477373123169, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8550097346305847, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 18.677566528320312, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -7.575146198272705, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 29.282791137695312, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.044503092765808105, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.23529298603534698, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.23072893917560577, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.04439074173569679, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2712540626525879, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.08485856652259827, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 3.550028085708618, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0531311109662056, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 7.043406963348389, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 601.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.4309736490249634, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.996856689453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 7.9134392738342285, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 7.727206230163574, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5323148965835571, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.25959014892578, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.11457109451293945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.658489227294922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2533256411552429, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.615750312805176, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1702.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.19894075393676758, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -17.19533920288086, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 22.322967529296875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.1228840351104736, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.6741649508476257, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08357564359903336, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 21.949264526367188, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -6.2199015617370605, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 43.8353271484375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.036901794373989105, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20942999422550201, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.21063092350959778, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.004942667204886675, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.251563161611557, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0023130301851779222, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.5314580798149109, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.015570314601063728, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.4083219766616821, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 9079.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7332767248153687, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.867337703704834, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.9761040210723877, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.7891526222229004, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.43142929673194885, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.487104415893555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.021400878205895424, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.462429404258728, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.12102392315864563, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 4.113752365112305, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19345.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4235200881958008, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 12.20568561553955, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.900527954101562, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 18.899032592773438, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -5.845510005950928, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.4231160879135132, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.500914573669434, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.2780745029449463, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 16.658632278442383, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.037420544773340225, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.212787464261055, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.2127508670091629, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0008428853470832109, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.25379619002342224, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.012625893577933311, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.989250898361206, "metrics/particle/momentum/all/accepted/pz_bias": 0.017340660095214844, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.7053799629211426, "metrics/particle/momentum/all/accepted/samples": 13702.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.8014515042304993, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.315683841705322, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.303313255310059, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4.1158294677734375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.4378914535045624, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.362201690673828, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.03314486891031265, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.9383950233459473, "metrics/particle/momentum/all/assigned/pz_bias": -0.15975642204284668, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.938722610473633, "metrics/particle/momentum/all/assigned/samples": 29537.0, "metrics/particle/momentum/all/efficiency": 0.38712775707244873, "metrics/particle/momentum/all/event/energy_bias": 1.2052326202392578, "metrics/particle/momentum/all/event/p3_error_mean": 49.97396469116211, "metrics/particle/momentum/all/event/pt_vector_error_mean": 22.85460090637207, "metrics/particle/momentum/all/event/px_bias": -7.858152866363525, "metrics/particle/momentum/all/event/py_bias": 2.194550037384033, "metrics/particle/momentum/all/event/pz_abs_mean": 38.62014389038086, "metrics/particle/momentum/all/event/pz_bias": -16.073122024536133, "metrics/particle/momentum/all/event/pz_residual_rms": 53.688392639160156, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04891372472047806, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2332228571176529, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22676779329776764, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03694310411810875, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2664264142513275, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.014673514291644096, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.1496860980987549, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0022280921693891287, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.699648380279541, "metrics/particle/momentum/class_1/accepted/samples": 6815.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.6808764338493347, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.9712789058685303, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 3.1285414695739746, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.8903608322143555, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3953852653503418, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 14.594732284545898, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.036319613456726074, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 2.030543327331543, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.18892772495746613, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.851194858551025, "metrics/particle/momentum/class_1/assigned/samples": 14042.0, "metrics/particle/momentum/class_1/efficiency": 0.408401757478714, "metrics/particle/momentum/class_1/event/energy_bias": 22.01830291748047, "metrics/particle/momentum/class_1/event/p3_error_mean": 40.08888244628906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 22.009441375732422, "metrics/particle/momentum/class_1/event/px_bias": -4.742757797241211, "metrics/particle/momentum/class_1/event/py_bias": -1.469908595085144, "metrics/particle/momentum/class_1/event/pz_abs_mean": 29.260547637939453, "metrics/particle/momentum/class_1/event/pz_bias": -14.73031234741211, "metrics/particle/momentum/class_1/event/pz_residual_rms": 40.1231575012207, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04252547398209572, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.25102561712265015, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.24681521952152252, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08832736313343048, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2862313985824585, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028629856184124947, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8749477863311768, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.06782545149326324, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.47463846206665, "metrics/particle/momentum/class_2/accepted/samples": 978.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8084450364112854, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8196321725845337, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.168875813484192, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7735968828201294, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3548067510128021, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.402731895446777, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05649157613515854, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8263583183288574, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.037347808480262756, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.140489101409912, "metrics/particle/momentum/class_2/assigned/samples": 2018.0, "metrics/particle/momentum/class_2/efficiency": 0.3285186290740967, "metrics/particle/momentum/class_2/event/energy_bias": -42.65555953979492, "metrics/particle/momentum/class_2/event/p3_error_mean": 31.92270278930664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.961685180664062, "metrics/particle/momentum/class_2/event/px_bias": -0.2525797486305237, "metrics/particle/momentum/class_2/event/py_bias": 0.177365243434906, "metrics/particle/momentum/class_2/event/pz_abs_mean": 24.348770141601562, "metrics/particle/momentum/class_2/event/pz_bias": -0.8162629008293152, "metrics/particle/momentum/class_2/event/pz_residual_rms": 37.62767028808594, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0224616676568985, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18045790493488312, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.189154252409935, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.061754703521728516, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.23045937716960907, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.005624890327453613, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.5825257301330566, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03450421616435051, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 2.108999729156494, "metrics/particle/momentum/class_3/accepted/samples": 5689.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.9337930679321289, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 6.258059501647949, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 6.027379512786865, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.931857109069824, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4976983368396759, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.77977752685547, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02455117367208004, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.6566390991210938, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.13942144811153412, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.735645294189453, "metrics/particle/momentum/class_3/assigned/samples": 13034.0, "metrics/particle/momentum/class_3/efficiency": 0.37410402297973633, "metrics/particle/momentum/class_3/event/energy_bias": 35.51704406738281, "metrics/particle/momentum/class_3/event/p3_error_mean": 30.633163452148438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 16.77884864807129, "metrics/particle/momentum/class_3/event/px_bias": -1.9452842473983765, "metrics/particle/momentum/class_3/event/py_bias": 1.534652829170227, "metrics/particle/momentum/class_3/event/pz_abs_mean": 21.684431076049805, "metrics/particle/momentum/class_3/event/pz_bias": -0.3538220226764679, "metrics/particle/momentum/class_3/event/pz_residual_rms": 35.15764236450195, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04528103023767471, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.24892392754554749, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.23697476089000702, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.12483920156955719, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2717910706996918, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.054054055362939835, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.589292526245117, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2626214027404785, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 4.9318156242370605, "metrics/particle/momentum/class_4/accepted/samples": 148.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.8153567910194397, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.333691596984863, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.664849281311035, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.382143020629883, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.43478259444236755, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.52796173095703, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.059006210416555405, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.707230567932129, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.16363413631916046, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.425739765167236, "metrics/particle/momentum/class_4/assigned/samples": 322.0, "metrics/particle/momentum/class_4/efficiency": 0.4065934121608734, "metrics/particle/momentum/class_4/event/energy_bias": -7.64935302734375, "metrics/particle/momentum/class_4/event/p3_error_mean": 7.2023725509643555, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 5.516716957092285, "metrics/particle/momentum/class_4/event/px_bias": -0.14468447864055634, "metrics/particle/momentum/class_4/event/py_bias": 1.4535276889801025, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.640577554702759, "metrics/particle/momentum/class_4/event/pz_bias": 0.08578944206237793, "metrics/particle/momentum/class_4/event/pz_residual_rms": 10.9092378616333, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04601863771677017, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23931990563869476, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23796862363815308, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08672342449426651, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2793964743614197, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0694444477558136, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.620868682861328, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6765215992927551, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.4415764808654785, "metrics/particle/momentum/class_5/accepted/samples": 72.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.384806364774704, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.3772467374801636, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9110843539237976, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4200136363506317, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.32231405377388, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.915989637374878, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.13223139941692352, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.739973545074463, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.9960735440254211, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.998621940612793, "metrics/particle/momentum/class_5/assigned/samples": 121.0, "metrics/particle/momentum/class_5/efficiency": 0.4528301954269409, "metrics/particle/momentum/class_5/event/energy_bias": -6.025203227996826, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.403782844543457, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.26151180267334, "metrics/particle/momentum/class_5/event/px_bias": -0.7728466391563416, "metrics/particle/momentum/class_5/event/py_bias": 0.4989129602909088, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.6867682933807373, "metrics/particle/momentum/class_5/event/pz_bias": -0.2585137188434601, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.452847480773926, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7667871713638306, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7752026319503784, "metrics/set_slot/angular_recall_all_dr0p10": 0.8096388578414917, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5230656266212463, "metrics/set_slot/soft_count_bias_mean": -2.050971031188965}, "step_train_time": 4.850788354873657, "step_valid_time": 4701.985617876053, "step_total_time": 4706.836406707764, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1454745871688519, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.42244278885136377, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2370, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1966, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7829534192269574, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7876370887337986, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7852882703777336, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2123629112662014, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6494879418566237, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6533732136922565, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6514247846255798, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7829534192269574, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0930096258145672, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3881428045591646, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2178, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1728, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1528, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7639257294429708, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7933884297520661, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7783783783783784, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2066115702479339, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6755083996463307, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7015610651974288, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6882882882882883, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7639257294429708, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0126786591323538, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3845943559587055, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 912, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 810, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7080745341614907, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7755102040816326, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7402597402597403, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.22448979591836735, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6288819875776398, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6887755102040817, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6574675324675324, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7080745341614907} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_20000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..84cee288d6e9209257639136190fdfb416248702 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9475080538184575, "train": {"Classification_binary": 0.1902996301651001, "Cardinality": 0.13685999810695648, "Classification": 0.11021541804075241, "Regression_pt": 0.17782895267009735, "Regression_eta": 0.002416100585833192, "Regression_sin_phi": 0.0010899072512984276, "Regression_cos_phi": 0.0012170764384791255, "Regression_energy": 0.17447414994239807, "HitGrouping": 0.21690937876701355, "Auxiliary": 0.2712019085884094, "Total": 1.3130539655685425, "Total_unweighted": 0.6575412350939587}, "train_group": {}, "valid": {"Classification_binary": 0.21268383661905924, "Cardinality": 0.14379505316416422, "Classification": 0.1071348786354065, "Regression_pt": 0.1752558946609497, "Regression_eta": 0.0021732614065210023, "Regression_sin_phi": 0.001135929332425197, "Regression_cos_phi": 0.001110433756063382, "Regression_energy": 0.17379528284072876, "HitGrouping": 0.21889869372049967, "Auxiliary": 0.27428040901819867, "Total": 1.3310224215189617, "Total_unweighted": 0.6732895172511538, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7737548351287842, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.40243396162986755, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41617077589035034, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.39807406067848206, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.39958107471466064, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5159157514572144, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.703921377658844, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3537076413631439, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42494070529937744, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.34952592849731445, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3746606409549713, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6065740585327148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7921764850616455, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.39885076880455017, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.387201726436615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.393257200717926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3862907588481903, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.48724305629730225, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7783324122428894, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3959546685218811, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.39990881085395813, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.39088472723960876, "metrics/hit_grouping/calorimeter/all/object_purity": 0.38952431082725525, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5071732401847839, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025797268375754356, "metrics/hit_grouping/calorimeter/background_recall": 0.6312299370765686, "metrics/hit_grouping/calorimeter/completeness": 0.45136380195617676, "metrics/hit_grouping/calorimeter/energy_completeness": 0.47330009937286377, "metrics/hit_grouping/calorimeter/energy_purity": 0.4678124785423279, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.45927250385284424, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8001032471656799, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.40811872482299805, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.31949836015701294, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.40650397539138794, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4087122678756714, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.45320990681648254, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7734431624412537, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.41836369037628174, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4422610402107239, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.41116976737976074, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4111795127391815, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5275352001190186, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7373459935188293, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.34685495495796204, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5259764790534973, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3366849422454834, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.32634299993515015, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.600176990032196, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7629595994949341, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.42753881216049194, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4838990867137909, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4221591353416443, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4241267740726471, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5451261401176453, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7486627101898193, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5034326910972595, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6405023336410522, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4912343919277191, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.49007585644721985, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5536368489265442, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7986558079719543, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.34824636578559875, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.27882683277130127, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.34442171454429626, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.33945757150650024, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.46042588353157043, "metrics/hit_grouping/calorimeter/purity": 0.4447195529937744, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8543052077293396, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07367100566625595, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15076923370361328, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07375457882881165, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03601570427417755, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04923076927661896, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7917637825012207, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1672043651342392, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11972633749246597, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16523905098438263, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1571696400642395, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.366020530462265, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7759405970573425, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.41598063707351685, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4213339388370514, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4106222987174988, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4104231297969818, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5250524878501892, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7213610410690308, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4444112479686737, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45697325468063354, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4687081575393677, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6071428656578064, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7721168398857117, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.31295570731163025, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4822089970111847, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4923817217350006, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4699943959712982, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6693527102470398, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45262661576271057, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4002281129360199, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40413698554039, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6717334985733032, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6944457292556763, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4382675290107727, "metrics/hit_grouping/tracker/all/object_completeness": 0.4250679612159729, "metrics/hit_grouping/tracker/all/object_purity": 0.4319780766963959, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6343283653259277, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0061093708500266075, "metrics/hit_grouping/tracker/background_recall": 0.029674820601940155, "metrics/hit_grouping/tracker/completeness": 0.4413487911224365, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4986051917076111, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7004960775375366, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42277905344963074, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4762575626373291, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4952206611633301, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6302505731582642, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6896039247512817, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46049875020980835, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40918341279029846, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.411081463098526, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6434418559074402, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6861591339111328, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4490291178226471, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32831162214279175, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3137907087802887, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6336973905563354, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7257273197174072, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46198779344558716, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4528399705886841, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4452885389328003, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5898634195327759, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7475910186767578, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4513981342315674, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46233025193214417, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.45556384325027466, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5492676496505737, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6443489789962769, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4050573706626892, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3823363184928894, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41049063205718994, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.707211434841156, "metrics/hit_grouping/tracker/purity": 0.42119476199150085, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8742557764053345, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014769765548408031, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.037994518876075745, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04170844703912735, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08644656836986542, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6578391194343567, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2244686335325241, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2332373708486557, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.23837359249591827, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5878693461418152, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6741806864738464, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5201581716537476, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4995077848434448, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5070541501045227, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7140664458274841, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.46614664793014526, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5255722403526306, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5759591460227966, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5976071953773499, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.019580719992518425, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06680241972208023, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.965068817138672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2665872573852539, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1361474096775055, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8445191979408264, "metrics/particle/class_1/pid_accuracy": 0.9573356509208679, "metrics/particle/class_2/efficiency": 0.3713259994983673, "metrics/particle/class_2/pid_accuracy": 0.5429003238677979, "metrics/particle/class_3/efficiency": 0.6678823232650757, "metrics/particle/class_3/pid_accuracy": 0.9582744240760803, "metrics/particle/class_4/efficiency": 0.6881616711616516, "metrics/particle/class_4/pid_accuracy": 0.31748253107070923, "metrics/particle/class_5/efficiency": 0.8999999761581421, "metrics/particle/class_5/pid_accuracy": 0.3911111056804657, "metrics/particle/count/bias_mean": -3.5846354961395264, "metrics/particle/count/mae": 5.359375, "metrics/particle/count/prediction_mean": 66.47135162353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.10091721266508102, "metrics/particle/event/energy_response_mean": 0.928519070148468, "metrics/particle/event/met_abs_error": 8.83314323425293, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07673364132642746, "metrics/particle/event/scalar_pt_response_mean": 0.962898313999176, "metrics/particle/event/vector_pt_closure": 0.07103896141052246, "metrics/particle/matched/delta_eta_abs_mean": 0.019019199535250664, "metrics/particle/matched/delta_phi_abs_mean": 0.016393238678574562, "metrics/particle/matched/delta_r_mean": 0.028040561825037003, "metrics/particle/matched/energy_relative_abs_mean": 0.15631254017353058, "metrics/particle/matched/pid_accuracy": 0.9310886859893799, "metrics/particle/matched/pt_relative_abs_mean": 0.1558866649866104, "metrics/particle/matching/duplicate_fraction": 0.07217434048652649, "metrics/particle/matching/efficiency": 0.7275523543357849, "metrics/particle/matching/f1": 0.7466548681259155, "metrics/particle/matching/purity": 0.7667874693870544, "metrics/particle/matching/target_coverage_dr0p05": 0.6491924524307251, "metrics/particle/matching/target_coverage_dr0p10": 0.782883882522583, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5277586579322815, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7779492139816284, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026762401685118675, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1524878740310669, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15193678438663483, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0024063438177108765, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19219498336315155, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009650025516748428, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.798844575881958, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.021865176036953926, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.143718957901001, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7772.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.34076017141342163, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0747648477554321, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.335537075996399, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.146268606185913, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16520865261554718, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.773843765258789, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019507290795445442, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2179843187332153, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07437708973884583, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3004584312438965, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9945.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7106803059577942, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -6.306396484375, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.827699661254883, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.7015557289123535, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.060208454728126526, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.4951121211051941, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.362519264221191, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.347359299659729, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.070656776428223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026631850749254227, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15658196806907654, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1598440557718277, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03634147718548775, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20126144587993622, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023134760558605194, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4511336088180542, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.014275404624640942, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.620467185974121, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1729.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5453667640686035, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9754136800765991, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.6941871643066406, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.524613857269287, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21713565289974213, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 16.229549407958984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05459890887141228, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4882290363311768, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.026056788861751556, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.57852840423584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2381.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5723270177841187, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.517237663269043, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.371502876281738, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.777603268623352, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14761804044246674, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.011584779247641563, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.162683486938477, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.18862266838550568, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 28.567174911499023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028453022241592407, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15601663291454315, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15595024824142456, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.009699969552457333, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19572919607162476, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010408376110717654, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.28236162662506104, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.015904543921351433, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8980675935745239, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16333.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.27234339714050293, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3701910972595215, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1009016036987305, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8962126970291138, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1621764600276947, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.111562728881836, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0048109302297234535, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4897850453853607, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007119056303054094, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6701973676681519, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20786.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7619069814682007, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.5440285205841064, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.066797256469727, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.150538444519043, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.2376807928085327, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.7434778213500977, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.513646602630615, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4008895754814148, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.901932716369629, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027822522446513176, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15499287843704224, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1550033986568451, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.004424297250807285, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19504986703395844, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005109545774757862, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5159651637077332, "metrics/particle/momentum/all/accepted/pz_bias": 0.01567790098488331, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6642459630966187, "metrics/particle/momentum/all/accepted/samples": 25834.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.31252437829971313, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.253074049949646, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2859421968460083, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0884097814559937, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16703914105892181, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.291694641113281, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012805025093257427, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8521985411643982, "metrics/particle/momentum/all/assigned/pz_bias": 0.024934029206633568, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.852146863937378, "metrics/particle/momentum/all/assigned/samples": 33112.0, "metrics/particle/momentum/all/efficiency": 0.7298977375030518, "metrics/particle/momentum/all/event/energy_bias": -20.36766242980957, "metrics/particle/momentum/all/event/p3_error_mean": 26.183940887451172, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.342424392700195, "metrics/particle/momentum/all/event/px_bias": -1.445507287979126, "metrics/particle/momentum/all/event/py_bias": 2.2501749992370605, "metrics/particle/momentum/all/event/pz_abs_mean": 19.787113189697266, "metrics/particle/momentum/all/event/pz_bias": 0.7578468918800354, "metrics/particle/momentum/all/event/pz_residual_rms": 31.120710372924805, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03061748296022415, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14184589684009552, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14139792323112488, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.013900737278163433, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17882601916790009, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005225988570600748, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5258643627166748, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0312502421438694, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6063086986541748, "metrics/particle/momentum/class_1/accepted/samples": 14160.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1636849343776703, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33111345767974854, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.40964555740356445, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2262030839920044, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08792635053396225, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.8706493377685547, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008897677063941956, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6880038380622864, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04047686234116554, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2297940254211426, "metrics/particle/momentum/class_1/assigned/samples": 16184.0, "metrics/particle/momentum/class_1/efficiency": 0.8485647439956665, "metrics/particle/momentum/class_1/event/energy_bias": -0.5469645261764526, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.693195343017578, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.543015480041504, "metrics/particle/momentum/class_1/event/px_bias": 1.116184115409851, "metrics/particle/momentum/class_1/event/py_bias": 1.350654125213623, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.046863555908203, "metrics/particle/momentum/class_1/event/pz_bias": -0.7656430006027222, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.050172805786133, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03457524627447128, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2005767524242401, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20210324227809906, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0508696548640728, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24330800771713257, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0272479560226202, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.782143235206604, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.009295814670622349, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.257165908813477, "metrics/particle/momentum/class_2/accepted/samples": 1101.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8168725967407227, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7976773381233215, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.163812279701233, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6921526789665222, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3617415428161621, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.300917148590088, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05094951391220093, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6548011302948, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0913872942328453, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.734413146972656, "metrics/particle/momentum/class_2/assigned/samples": 2159.0, "metrics/particle/momentum/class_2/efficiency": 0.36983540654182434, "metrics/particle/momentum/class_2/event/energy_bias": -22.67617416381836, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.698482513427734, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.756280899047852, "metrics/particle/momentum/class_2/event/px_bias": -1.5971778631210327, "metrics/particle/momentum/class_2/event/py_bias": 1.0199882984161377, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.664337158203125, "metrics/particle/momentum/class_2/event/pz_bias": -0.182587131857872, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.840639114379883, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023202722892165184, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1683332324028015, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16898030042648315, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014397051185369492, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21032856404781342, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021619496401399374, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.330841600894928, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0161688681691885, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0645283460617065, "metrics/particle/momentum/class_3/accepted/samples": 10176.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4062981903553009, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.321290969848633, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2553818225860596, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0823934078216553, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22734598815441132, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.541862487792969, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01093279104679823, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7358514666557312, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.010371880605816841, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7583649158477783, "metrics/particle/momentum/class_3/assigned/samples": 14269.0, "metrics/particle/momentum/class_3/efficiency": 0.6691654920578003, "metrics/particle/momentum/class_3/event/energy_bias": 6.167172431945801, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.273941040039062, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.943087100982666, "metrics/particle/momentum/class_3/event/px_bias": -0.6150040626525879, "metrics/particle/momentum/class_3/event/py_bias": -0.23181580007076263, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.071394920349121, "metrics/particle/momentum/class_3/event/pz_bias": 0.971786379814148, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.436857223510742, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.030950287356972694, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15259574353694916, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14791151881217957, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0014983355067670345, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1910226196050644, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01606425642967224, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1958738565444946, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.40429753065109253, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.7650954723358154, "metrics/particle/momentum/class_4/accepted/samples": 249.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3773103356361389, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.669116258621216, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.575148582458496, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.4268991947174072, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2215743511915207, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.70508575439453, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02332361601293087, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4627907276153564, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.20681558549404144, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.9825241565704346, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.6840659379959106, "metrics/particle/momentum/class_4/event/energy_bias": -1.6164301633834839, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.5377109050750732, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.5566649436950684, "metrics/particle/momentum/class_4/event/px_bias": -0.06360027194023132, "metrics/particle/momentum/class_4/event/py_bias": -0.07124388962984085, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9599350690841675, "metrics/particle/momentum/class_4/event/pz_bias": 0.4743967652320862, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.141258239746094, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.022558284923434258, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1605253666639328, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15726079046726227, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.051261186599731445, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19435612857341766, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.013513513840734959, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.734100580215454, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.10911554843187332, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3160171508789062, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05551636964082718, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1901462972164154, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18741703033447266, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0312053132802248, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04458598792552948, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.25797706842422485, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03821656107902527, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.22947359085083, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.32032808661460876, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.976760387420654, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.9308176040649414, "metrics/particle/momentum/class_5/event/energy_bias": -1.6952699422836304, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7451605796813965, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0348284244537354, "metrics/particle/momentum/class_5/event/px_bias": -0.2859092950820923, "metrics/particle/momentum/class_5/event/py_bias": 0.18259213864803314, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4375462532043457, "metrics/particle/momentum/class_5/event/pz_bias": 0.25989389419555664, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.834754943847656, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4438541829586029, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4448450803756714, "metrics/set_slot/angular_recall_all_dr0p10": 0.9309518337249756, "metrics/set_slot/angular_recall_selected_dr0p10": 0.771509051322937, "metrics/set_slot/soft_count_bias_mean": 0.5005841851234436}, "step_train_time": 4.723880767822266, "step_valid_time": 51280.97326016426, "step_total_time": 51285.6971411705, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9911397024960018, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17875960911144584, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2973, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2816, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2786, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9302940204823258, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9471913891691893, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9386666666666666, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05280861083081063, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203832177073009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9371005718129836, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9286666666666666, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9302940204823258, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.968700446062831, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17445655969438656, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2087, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2076, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9226348364279399, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9350358422939068, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9287939474855362, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06496415770609319, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9177718832891246, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9301075268817204, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9238985313751669, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9226348364279399, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9731136987503569, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18426539715940826, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1252, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1161, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9013975155279503, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9273162939297125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9141732283464566, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07268370607028754, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8920807453416149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9177316293929713, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9047244094488189, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9013975155279503} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_21000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..0fd30bd5c85c64bc3a4cecc3dfec1104f23206e0 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9948834565093804, "train": {"Classification_binary": 0.20077969133853912, "Cardinality": 0.15205422043800354, "Classification": 0.1140841394662857, "Regression_pt": 0.19491256773471832, "Regression_eta": 0.0019652722403407097, "Regression_sin_phi": 0.0010704310843721032, "Regression_cos_phi": 0.0010772208916023374, "Regression_energy": 0.19169288873672485, "HitGrouping": 0.24100260436534882, "Auxiliary": 0.2849082052707672, "Total": 1.3680461645126343, "Total_unweighted": 0.7055822114925832}, "train_group": {}, "valid": {"Classification_binary": 0.226671834786733, "Cardinality": 0.17078510920206705, "Classification": 0.11201671759287517, "Regression_pt": 0.19487218062082926, "Regression_eta": 0.0022435771922270456, "Regression_sin_phi": 0.001181847881525755, "Regression_cos_phi": 0.001194469475497802, "Regression_energy": 0.19172736008961996, "HitGrouping": 0.24222383896509805, "Auxiliary": 0.29862238963445026, "Total": 1.4534072875976562, "Total_unweighted": 0.729907987639308, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7097402215003967, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.34260743856430054, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4491357207298279, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3298369348049164, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3459056317806244, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6458864808082581, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6351697444915771, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2818358242511749, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.43713995814323425, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.26599013805389404, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3173621594905853, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7309386730194092, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7310323715209961, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.35550135374069214, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.43151533603668213, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3434685468673706, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34616756439208984, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5958062410354614, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7155822515487671, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3447447121143341, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.43768996000289917, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3321325182914734, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3434995412826538, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6240425705909729, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02628481201827526, "metrics/hit_grouping/calorimeter/background_recall": 0.6333890557289124, "metrics/hit_grouping/calorimeter/completeness": 0.38155683875083923, "metrics/hit_grouping/calorimeter/energy_completeness": 0.41700640320777893, "metrics/hit_grouping/calorimeter/energy_purity": 0.41222134232521057, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.510729193687439, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7406228184700012, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.356629341840744, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.35359808802604675, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.34768855571746826, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.36521652340888977, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5666766166687012, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7080393433570862, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3645303547382355, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.49017468094825745, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.35003960132598877, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3619208335876465, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6493207216262817, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.670439600944519, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.29897022247314453, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5609910488128662, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2805445194244385, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2783350944519043, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7191252708435059, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.6934149861335754, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3651524782180786, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.53422611951828, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.35069164633750916, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3619038164615631, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6756150722503662, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6807941198348999, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.45897743105888367, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6713762283325195, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.43324407935142517, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4450034201145172, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6875981092453003, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7438650727272034, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3081875145435333, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3040042817592621, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.29924169182777405, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.31072500348091125, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5604660511016846, "metrics/hit_grouping/calorimeter/purity": 0.3761284351348877, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.838690459728241, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07621048390865326, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1830769181251526, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07538461685180664, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.023202072829008102, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04461538419127464, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7532749772071838, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14238348603248596, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11801595985889435, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1417616456747055, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14149056375026703, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3865450322628021, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7107903361320496, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3621072769165039, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.46150314807891846, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.34855422377586365, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.36194518208503723, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6500524878501892, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.720535159111023, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44216975569725037, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4414770007133484, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4474354088306427, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.619994044303894, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7684805989265442, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34604600071907043, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46385666728019714, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46798211336135864, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4806506037712097, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6608558297157288, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45317792892456055, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3806045651435852, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3845943808555603, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6915019154548645, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.688782811164856, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44067636132240295, "metrics/hit_grouping/tracker/all/object_completeness": 0.40685608983039856, "metrics/hit_grouping/tracker/all/object_purity": 0.41147953271865845, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6511430144309998, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004883226472884417, "metrics/hit_grouping/tracker/background_recall": 0.020967837423086166, "metrics/hit_grouping/tracker/completeness": 0.42588770389556885, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5247201919555664, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7007395625114441, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4276081919670105, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4603717029094696, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4701993465423584, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6424601078033447, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6791682839393616, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4639253616333008, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3922763168811798, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3950382173061371, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6615267395973206, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6724556684494019, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4449838101863861, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3035496175289154, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2985984683036804, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.659385085105896, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7134760618209839, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46207764744758606, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42633309960365295, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4256680905818939, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6131380200386047, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7183945178985596, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45006656646728516, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4347892105579376, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.441600501537323, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6005326509475708, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6514070630073547, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4111449420452118, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37657082080841064, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.387699156999588, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7095528244972229, "metrics/hit_grouping/tracker/purity": 0.40593889355659485, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8838139772415161, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014335360378026962, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02351471781730652, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.026985006406903267, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.09252823889255524, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.658436119556427, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19699326157569885, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.18992914259433746, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20874260365962982, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6003110408782959, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.665736734867096, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5263561606407166, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48364630341529846, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.48681071400642395, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7328374981880188, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3995503783226013, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4622567296028137, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.49807995557785034, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5812376737594604, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08617696166038513, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05043293535709381, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.87902069091797, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29000580310821533, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1595660001039505, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8243569731712341, "metrics/particle/class_1/pid_accuracy": 0.955500066280365, "metrics/particle/class_2/efficiency": 0.3645950257778168, "metrics/particle/class_2/pid_accuracy": 0.492000013589859, "metrics/particle/class_3/efficiency": 0.6375688910484314, "metrics/particle/class_3/pid_accuracy": 0.947441041469574, "metrics/particle/class_4/efficiency": 0.6823869347572327, "metrics/particle/class_4/pid_accuracy": 0.3060648739337921, "metrics/particle/class_5/efficiency": 0.8659999966621399, "metrics/particle/class_5/pid_accuracy": 0.3556582033634186, "metrics/particle/count/bias_mean": -2.5169270038604736, "metrics/particle/count/mae": 5.055989742279053, "metrics/particle/count/prediction_mean": 67.5390625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09030898660421371, "metrics/particle/event/energy_response_mean": 0.9643808007240295, "metrics/particle/event/met_abs_error": 11.231583595275879, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07271627336740494, "metrics/particle/event/scalar_pt_response_mean": 0.9888010025024414, "metrics/particle/event/vector_pt_closure": 0.08007043600082397, "metrics/particle/matched/delta_eta_abs_mean": 0.017604032531380653, "metrics/particle/matched/delta_phi_abs_mean": 0.016652459278702736, "metrics/particle/matched/delta_r_mean": 0.027107983827590942, "metrics/particle/matched/energy_relative_abs_mean": 0.166419118642807, "metrics/particle/matched/pid_accuracy": 0.9229550957679749, "metrics/particle/matched/pt_relative_abs_mean": 0.16926154494285583, "metrics/particle/matching/duplicate_fraction": 0.08108733594417572, "metrics/particle/matching/efficiency": 0.7041800618171692, "metrics/particle/matching/f1": 0.7170611023902893, "metrics/particle/matching/purity": 0.7304221987724304, "metrics/particle/matching/target_coverage_dr0p05": 0.6414419412612915, "metrics/particle/matching/target_coverage_dr0p10": 0.7725870013237, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4801962673664093, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7592606544494629, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025694603100419044, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1635335087776184, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16919076442718506, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02849036268889904, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20973026752471924, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01141552533954382, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9236056804656982, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.16913796961307526, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.46805739402771, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7446.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.37973180413246155, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4390522241592407, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6997432708740234, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5287542343139648, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20220807194709778, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.099093437194824, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.023970559239387512, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3821128606796265, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.2198953479528427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.593378782272339, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10054.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.680870532989502, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0301475524902344, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.080761909484863, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.235548973083496, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.4026012420654297, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8264718055725098, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.489434242248535, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 4.151058673858643, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.610620498657227, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025074364617466927, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16999611258506775, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1720772534608841, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02803708054125309, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21131671965122223, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0317460335791111, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.743857979774475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.32164302468299866, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.350991725921631, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1701.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.578079104423523, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4118075370788574, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.1608235836029053, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.9998581409454346, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24318744242191315, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 27.222023010253906, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.061519406735897064, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.740212917327881, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.41392838954925537, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.680174827575684, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2422.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.563058614730835, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.472099304199219, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.366332054138184, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5162273645401, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.051735952496528625, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12135959416627884, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.209981918334961, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4606735110282898, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.53333282470703, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02785491943359375, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1650124341249466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16694095730781555, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01353803463280201, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20775337517261505, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0012071923119947314, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.30681169033050537, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03602191060781479, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9061512351036072, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15739.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3114035129547119, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.8357959985733032, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.4787392616271973, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.2867074012756348, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.18964116275310516, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.630438804626465, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006928185932338238, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6003320813179016, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03784365579485893, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.149441719055176, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20929.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7341978549957275, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.17941562831401825, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.934874534606934, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.026422500610352, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.386969089508057, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9206002354621887, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.6895527839660645, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.6430500745773315, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.559846878051758, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027018487453460693, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16491056978702545, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16796517372131348, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019002871587872505, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20859162509441376, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006348951254040003, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5895837545394897, "metrics/particle/momentum/all/accepted/pz_bias": 0.0953734964132309, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9067842960357666, "metrics/particle/momentum/all/accepted/samples": 24886.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3513035476207733, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6856459379196167, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6672136783599854, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4837673902511597, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1973057985305786, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.394657135009766, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016015566885471344, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9907771348953247, "metrics/particle/momentum/all/assigned/pz_bias": 0.11990397423505783, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1648619174957275, "metrics/particle/momentum/all/assigned/samples": 33405.0, "metrics/particle/momentum/all/efficiency": 0.7031134963035583, "metrics/particle/momentum/all/event/energy_bias": -8.322832107543945, "metrics/particle/momentum/all/event/p3_error_mean": 27.57765769958496, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.72999382019043, "metrics/particle/momentum/all/event/px_bias": 6.7378339767456055, "metrics/particle/momentum/all/event/py_bias": 1.868431568145752, "metrics/particle/momentum/all/event/pz_abs_mean": 20.4622802734375, "metrics/particle/momentum/all/event/pz_bias": 5.333434581756592, "metrics/particle/momentum/all/event/pz_residual_rms": 30.478588104248047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02901752106845379, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15779884159564972, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1614144891500473, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020634280517697334, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20073047280311584, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005956272128969431, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6180703639984131, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.13270026445388794, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8737356662750244, "metrics/particle/momentum/class_1/accepted/samples": 13767.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1803537756204605, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5204870700836182, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5763072371482849, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4080215394496918, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11290719360113144, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.967376708984375, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010940381325781345, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8244295120239258, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.17883841693401337, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5532643795013428, "metrics/particle/momentum/class_1/assigned/samples": 16270.0, "metrics/particle/momentum/class_1/efficiency": 0.8250134587287903, "metrics/particle/momentum/class_1/event/energy_bias": 5.391798973083496, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.580829620361328, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.192422866821289, "metrics/particle/momentum/class_1/event/px_bias": 4.913195610046387, "metrics/particle/momentum/class_1/event/py_bias": 0.588997483253479, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.115544319152832, "metrics/particle/momentum/class_1/event/pz_bias": 6.097565650939941, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.221914291381836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03750132396817207, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20398429036140442, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20341862738132477, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04349888488650322, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24529430270195007, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.033582091331481934, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9063555002212524, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.16783621907234192, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.370152950286865, "metrics/particle/momentum/class_2/accepted/samples": 1072.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8899587392807007, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8259470462799072, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1515536308288574, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7102884650230408, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3816554844379425, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.11392879486084, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06308724731206894, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8983278274536133, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.18754969537258148, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.357529163360596, "metrics/particle/momentum/class_2/assigned/samples": 2235.0, "metrics/particle/momentum/class_2/efficiency": 0.3600940406322479, "metrics/particle/momentum/class_2/event/energy_bias": -17.672060012817383, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.82428741455078, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.647052764892578, "metrics/particle/momentum/class_2/event/px_bias": 0.2098504900932312, "metrics/particle/momentum/class_2/event/py_bias": 0.6285378932952881, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.468040466308594, "metrics/particle/momentum/class_2/event/pz_bias": -1.371208906173706, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.929006576538086, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02304110862314701, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1700669676065445, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17282944917678833, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02443194016814232, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21451066434383392, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002794163301587105, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.36006367206573486, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.021492036059498787, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3077929019927979, "metrics/particle/momentum/class_3/accepted/samples": 9663.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4635346233844757, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.13181471824646, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.982205390930176, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.8216304779052734, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2638271152973175, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.844942092895508, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013688160106539726, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8533105850219727, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.037483684718608856, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.9686124324798584, "metrics/particle/momentum/class_3/assigned/samples": 14392.0, "metrics/particle/momentum/class_3/efficiency": 0.6354310512542725, "metrics/particle/momentum/class_3/event/energy_bias": 7.898136615753174, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.581995964050293, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.6689453125, "metrics/particle/momentum/class_3/event/px_bias": 1.4945164918899536, "metrics/particle/momentum/class_3/event/py_bias": 0.3365457355976105, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.66451358795166, "metrics/particle/momentum/class_3/event/pz_bias": -0.4193931818008423, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.43663787841797, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02840137481689453, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17338621616363525, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1752339005470276, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.023236412554979324, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21720223128795624, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.024390242993831635, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2991541624069214, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.5927982926368713, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.1304092407226562, "metrics/particle/momentum/class_4/accepted/samples": 246.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.37798383831977844, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.479248046875, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.19594144821167, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.033756971359253, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2542372941970825, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.254497528076172, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02259887009859085, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4269251823425293, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.18125462532043457, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.444840908050537, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.6758241653442383, "metrics/particle/momentum/class_4/event/energy_bias": -2.010112762451172, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.2273659706115723, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.224034070968628, "metrics/particle/momentum/class_4/event/px_bias": 0.05178820341825485, "metrics/particle/momentum/class_4/event/py_bias": 0.05119842663407326, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.976542353630066, "metrics/particle/momentum/class_4/event/pz_bias": 0.7272195816040039, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.306517124176025, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0221988707780838, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19468502700328827, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19249916076660156, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.045926909893751144, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23067791759967804, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.050724636763334274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.325397253036499, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09531909972429276, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.452638626098633, "metrics/particle/momentum/class_5/accepted/samples": 138.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04470546543598175, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.28539782762527466, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.29717037081718445, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.06746357679367065, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09090909361839294, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7804470062255859, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0714285746216774, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.725320339202881, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4733012914657593, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.326471328735352, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.8679245114326477, "metrics/particle/momentum/class_5/event/energy_bias": -1.9305932521820068, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0504307746887207, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.272000312805176, "metrics/particle/momentum/class_5/event/px_bias": 0.06848341226577759, "metrics/particle/momentum/class_5/event/py_bias": 0.26315194368362427, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6157231330871582, "metrics/particle/momentum/class_5/event/pz_bias": 0.2992522120475769, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.291935920715332, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4649336636066437, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.46921151876449585, "metrics/set_slot/angular_recall_all_dr0p10": 0.9252365231513977, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7637213468551636, "metrics/set_slot/soft_count_bias_mean": 2.192448616027832}, "step_train_time": 4.580043792724609, "step_valid_time": 53839.8919377327, "step_total_time": 53844.47198176384, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023057698506486, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2099428599398937, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2799, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2753, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9246778989098117, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9237623762376238, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.924219910846954, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07623762376237624, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9094813346547737, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9085808580858086, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9090308733696549, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9246778989098117, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0049611673441787, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19544769760956182, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2255, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2087, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2074, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9226348364279399, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9254988913525499, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9240646446756697, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07450110864745012, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9168877099911583, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9197339246119733, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9183086119105601, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9226348364279399, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050508564897052, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18747273937802755, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1140, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9021739130434783, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9056897895557288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9039284325165305, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09431021044427124, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8850931677018633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8885424785658612, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8868144690781797, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9021739130434783} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_22000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..7c074a5dcfb9ba78b5b737416d0d71fcb05e2e2d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0422588592003033, "train": {"Classification_binary": 0.24212481081485748, "Cardinality": 0.18490612506866455, "Classification": 0.1338738054037094, "Regression_pt": 0.19144771993160248, "Regression_eta": 0.002241872251033783, "Regression_sin_phi": 0.0011794471647590399, "Regression_cos_phi": 0.0011880185920745134, "Regression_energy": 0.19022785127162933, "HitGrouping": 0.2295284867286682, "Auxiliary": 0.3039087951183319, "Total": 1.4853076934814453, "Total_unweighted": 0.762283525429666}, "train_group": {}, "valid": {"Classification_binary": 0.22030401229858398, "Cardinality": 0.1504704157511393, "Classification": 0.10952999194463094, "Regression_pt": 0.19031351804733276, "Regression_eta": 0.0022230007064839206, "Regression_sin_phi": 0.0011504273085544507, "Regression_cos_phi": 0.0011480934917926788, "Regression_energy": 0.18856553236643472, "HitGrouping": 0.22666647036870322, "Auxiliary": 0.28480005264282227, "Total": 1.3853995005289714, "Total_unweighted": 0.7132345761638135, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7464805841445923, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.36726513504981995, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.420326828956604, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3617016673088074, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3736630082130432, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5766506195068359, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6888517141342163, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3438916504383087, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42460182309150696, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.33813709020614624, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.36414992809295654, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6540156006813049, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.776752769947052, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3748543858528137, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3974795937538147, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.36939844489097595, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.36194106936454773, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5280446410179138, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7591269612312317, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3696349561214447, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.40726444125175476, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.36411765217781067, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3659112751483917, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5549848079681396, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02547190710902214, "metrics/hit_grouping/calorimeter/background_recall": 0.6270164847373962, "metrics/hit_grouping/calorimeter/completeness": 0.4262581765651703, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44885510206222534, "metrics/hit_grouping/calorimeter/energy_purity": 0.44350093603134155, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4768103361129761, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7833003997802734, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.38234826922416687, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3234398365020752, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37980976700782776, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.38592636585235596, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4974619150161743, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7535589933395386, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.387606680393219, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4537845849990845, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3800276815891266, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.38315510749816895, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5758126974105835, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7137622237205505, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3239968419075012, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5362154245376587, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.314322829246521, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.30557602643966675, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6550372838973999, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7404105067253113, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3961924612522125, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.499781996011734, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.39139753580093384, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.39479050040245056, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6003114581108093, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7299588918685913, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4778616428375244, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6373626589775085, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.46511879563331604, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.46693024039268494, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6012558937072754, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.782980740070343, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3272349536418915, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2783581018447876, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.32186561822891235, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3219375014305115, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5003347992897034, "metrics/hit_grouping/calorimeter/purity": 0.41984331607818604, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8701116442680359, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07909812033176422, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1538461595773697, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07920145988464355, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0353812538087368, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05999999865889549, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.785205602645874, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.146298348903656, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10490307956933975, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14441248774528503, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1387714147567749, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3985176682472229, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7552614212036133, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38867291808128357, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4300563931465149, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.38282695412635803, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.38602033257484436, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5745343565940857, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7096517086029053, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44545724987983704, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4261278212070465, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.42948809266090393, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.636730432510376, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.766516387462616, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.335950642824173, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46172431111335754, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.467526912689209, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4985978603363037, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6632432341575623, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45089390873908997, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37758174538612366, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3717530071735382, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6867763996124268, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6866090893745422, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4394955635070801, "metrics/hit_grouping/tracker/all/object_completeness": 0.40001216530799866, "metrics/hit_grouping/tracker/all/object_purity": 0.3980673849582672, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6551105380058289, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006353379692882299, "metrics/hit_grouping/tracker/background_recall": 0.028194041922688484, "metrics/hit_grouping/tracker/completeness": 0.41830506920814514, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.523722231388092, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6928706765174866, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4202277958393097, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4460795223712921, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.45503559708595276, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6560364365577698, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.681255042552948, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4662097990512848, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38893428444862366, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.38222065567970276, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6573386788368225, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6783979535102844, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45388349890708923, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3095192015171051, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2884429693222046, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6506876945495605, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7166531085968018, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.47268152236938477, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.428276926279068, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4149981737136841, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6149352788925171, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7379751801490784, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.504660427570343, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4635032117366791, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4242718815803528, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5652463436126709, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6384376287460327, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38480448722839355, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35202643275260925, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3714032471179962, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7232498526573181, "metrics/hit_grouping/tracker/purity": 0.39930135011672974, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.879432737827301, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018679408356547356, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03237928822636604, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03255245462059975, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0951346680521965, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6583035588264465, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20114049315452576, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.206863671541214, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.21549570560455322, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6003110408782959, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6636305451393127, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5238179564476013, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4719603657722473, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.46852341294288635, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7374417185783386, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.42166391015052795, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4957420229911804, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5416268110275269, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5932052135467529, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0640634223818779, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06240040808916092, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.748388290405273, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2909054458141327, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16046562790870667, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8196873664855957, "metrics/particle/class_1/pid_accuracy": 0.9564052224159241, "metrics/particle/class_2/efficiency": 0.3689701557159424, "metrics/particle/class_2/pid_accuracy": 0.5208269953727722, "metrics/particle/class_3/efficiency": 0.6461931467056274, "metrics/particle/class_3/pid_accuracy": 0.9497028589248657, "metrics/particle/class_4/efficiency": 0.6814244389533997, "metrics/particle/class_4/pid_accuracy": 0.28672316670417786, "metrics/particle/class_5/efficiency": 0.8240000009536743, "metrics/particle/class_5/pid_accuracy": 0.3252427279949188, "metrics/particle/count/bias_mean": -4.545572757720947, "metrics/particle/count/mae": 5.85546875, "metrics/particle/count/prediction_mean": 65.51041412353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09052466601133347, "metrics/particle/event/energy_response_mean": 0.971210777759552, "metrics/particle/event/met_abs_error": 9.421737670898438, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07443622499704361, "metrics/particle/event/scalar_pt_response_mean": 0.9856686592102051, "metrics/particle/event/vector_pt_closure": 0.07285570353269577, "metrics/particle/matched/delta_eta_abs_mean": 0.017989134415984154, "metrics/particle/matched/delta_phi_abs_mean": 0.016200391575694084, "metrics/particle/matched/delta_r_mean": 0.02709914743900299, "metrics/particle/matched/energy_relative_abs_mean": 0.16830551624298096, "metrics/particle/matched/pid_accuracy": 0.9252208471298218, "metrics/particle/matched/pt_relative_abs_mean": 0.16514375805854797, "metrics/particle/matching/duplicate_fraction": 0.07241810858249664, "metrics/particle/matching/efficiency": 0.7058807015419006, "metrics/particle/matching/f1": 0.7295490503311157, "metrics/particle/matching/purity": 0.7548596858978271, "metrics/particle/matching/target_coverage_dr0p05": 0.6340352892875671, "metrics/particle/matching/target_coverage_dr0p10": 0.7661283016204834, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.49156180024147034, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7584707140922546, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025852037593722343, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1665799468755722, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1658284217119217, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01616271398961544, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20660191774368286, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011219447478652, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8863676190376282, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02766745164990425, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3012051582336426, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7487.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.34100884199142456, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1827119588851929, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.336159348487854, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1560416221618652, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17871424555778503, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.307826042175293, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02378755249083042, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3602285385131836, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0597282275557518, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5166804790496826, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9753.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6846196055412292, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.012242078781128, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.83039379119873, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.935624599456787, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.48053884506225586, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4297891855239868, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.231518745422363, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.48872196674346924, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.47852897644043, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0259255263954401, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17269550263881683, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17303228378295898, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.033916108310222626, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21527697145938873, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03575615584850311, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.8310112953186035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18035534024238586, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.683927059173584, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1706.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5206677913665771, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2500207424163818, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.4276299476623535, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.2624857425689697, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20937231183052063, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 32.055030822753906, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06663800776004791, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8798344135284424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.008166150189936161, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.389617919921875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2326.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5647136569023132, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.105271339416504, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.291598320007324, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6385722160339355, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.018638454377651215, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08267581462860107, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.104557991027832, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.5187554359436035, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.363805770874023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027513757348060608, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1682911217212677, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1634937822818756, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0037108887918293476, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2038038671016693, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009454178507439792, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.29810941219329834, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.016701998189091682, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8687026500701904, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15866.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2906453609466553, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.8031038045883179, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3652476072311401, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1720020771026611, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1741134226322174, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.797330856323242, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007179934997111559, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.598205029964447, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0187742430716753, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.4273834228515625, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20613.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7401221990585327, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7519049644470215, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.915875434875488, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.031341552734375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.7063778638839722, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9773357510566711, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.411065101623535, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.058057188987731934, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.069364547729492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026909153908491135, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16807971894741058, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1648406833410263, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009487532079219818, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20544207096099854, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00638493150472641, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5782251358032227, "metrics/particle/momentum/all/accepted/pz_bias": -0.031119609251618385, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.885094165802002, "metrics/particle/momentum/all/accepted/samples": 25059.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.32203614711761475, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5786712169647217, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5033059120178223, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3159761428833008, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17799462378025055, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.798083305358887, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01636485941708088, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9878748655319214, "metrics/particle/momentum/all/assigned/pz_bias": 0.006562170572578907, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.523134231567383, "metrics/particle/momentum/all/assigned/samples": 32692.0, "metrics/particle/momentum/all/efficiency": 0.7080013751983643, "metrics/particle/momentum/all/event/energy_bias": -7.3656086921691895, "metrics/particle/momentum/all/event/p3_error_mean": 24.965686798095703, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.540450096130371, "metrics/particle/momentum/all/event/px_bias": 2.2055552005767822, "metrics/particle/momentum/all/event/py_bias": -1.4898008108139038, "metrics/particle/momentum/all/event/pz_abs_mean": 18.50999641418457, "metrics/particle/momentum/all/event/pz_bias": -1.9719759225845337, "metrics/particle/momentum/all/event/pz_residual_rms": 29.029285430908203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02932734414935112, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15923041105270386, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1575501412153244, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.016240088269114494, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1969456672668457, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005888339597731829, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6063582897186279, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.028805548325181007, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8018975257873535, "metrics/particle/momentum/class_1/accepted/samples": 13756.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1689130961894989, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.48523402214050293, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5413115620613098, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.37810570001602173, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10490074753761292, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.604058742523193, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01066997554153204, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8098679184913635, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.008574812673032284, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.9289469718933105, "metrics/particle/momentum/class_1/assigned/samples": 16120.0, "metrics/particle/momentum/class_1/efficiency": 0.8243542909622192, "metrics/particle/momentum/class_1/event/energy_bias": 7.574704170227051, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.302106857299805, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.856398582458496, "metrics/particle/momentum/class_1/event/px_bias": 2.743760585784912, "metrics/particle/momentum/class_1/event/py_bias": 0.5306931138038635, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.682428359985352, "metrics/particle/momentum/class_1/event/pz_bias": 0.23384147882461548, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.9947566986084, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036013323813676834, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21319134533405304, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21105405688285828, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02654247358441353, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25186896324157715, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.041705284267663956, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0474112033843994, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11635519564151764, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.783751010894775, "metrics/particle/momentum/class_2/accepted/samples": 1079.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7661938071250916, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8054591417312622, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9368295073509216, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5181168913841248, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3549160659313202, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.0186591148376465, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06378896534442902, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9216575622558594, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.03637664392590523, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.041606426239014, "metrics/particle/momentum/class_2/assigned/samples": 2085.0, "metrics/particle/momentum/class_2/efficiency": 0.3624454140663147, "metrics/particle/momentum/class_2/event/energy_bias": -14.925354957580566, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.810274124145508, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.887652397155762, "metrics/particle/momentum/class_2/event/px_bias": -0.682616651058197, "metrics/particle/momentum/class_2/event/py_bias": -1.1648226976394653, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.166374206542969, "metrics/particle/momentum/class_2/event/pz_bias": -0.7350304126739502, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.70675277709961, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02251286245882511, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17459379136562347, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16923560202121735, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.003903498873114586, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.210675910115242, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002540908521041274, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34480375051498413, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02799980528652668, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2874935865402222, "metrics/particle/momentum/class_3/accepted/samples": 9839.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4353155493736267, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.9308857917785645, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.697531223297119, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5162692070007324, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.23561370372772217, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.691137313842773, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014797341078519821, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8750316500663757, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.026745116338133812, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.319549322128296, "metrics/particle/momentum/class_3/assigned/samples": 13989.0, "metrics/particle/momentum/class_3/efficiency": 0.6470046639442444, "metrics/particle/momentum/class_3/event/energy_bias": 2.719834804534912, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.867660522460938, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.598555564880371, "metrics/particle/momentum/class_3/event/px_bias": -0.21329568326473236, "metrics/particle/momentum/class_3/event/py_bias": -1.445217251777649, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.956555366516113, "metrics/particle/momentum/class_3/event/pz_bias": -1.482559323310852, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.321266174316406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.030429432168602943, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18828582763671875, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18094079196453094, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04172325134277344, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21924351155757904, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016129031777381897, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1945075988769531, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3193817138671875, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.65864896774292, "metrics/particle/momentum/class_4/accepted/samples": 248.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.32647407054901123, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.0949621200561523, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0066308975219727, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8716731071472168, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22543352842330933, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.363031387329102, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02601156011223793, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4319169521331787, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3531969487667084, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.771022319793701, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.6813187003135681, "metrics/particle/momentum/class_4/event/energy_bias": -1.3759689331054688, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.721467971801758, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.613262176513672, "metrics/particle/momentum/class_4/event/px_bias": 0.38853904604911804, "metrics/particle/momentum/class_4/event/py_bias": 0.30357101559638977, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1821463108062744, "metrics/particle/momentum/class_4/event/pz_bias": 0.15935634076595306, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.87106990814209, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021755971014499664, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19693009555339813, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18812766671180725, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0420813113451004, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2263842523097992, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.036496348679065704, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8303658962249756, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.45070433616638184, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4504690170288086, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033047568053007126, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.24702207744121552, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24198125302791595, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0074546076357364655, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09210526198148727, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3229924142360687, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09210526198148727, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.7145836353302, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4456705152988434, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.761462688446045, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.8616352081298828, "metrics/particle/momentum/class_5/event/energy_bias": -1.358823299407959, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0814151763916016, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.280811071395874, "metrics/particle/momentum/class_5/event/px_bias": -0.03083179146051407, "metrics/particle/momentum/class_5/event/py_bias": 0.28597524762153625, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.688274621963501, "metrics/particle/momentum/class_5/event/pz_bias": -0.1475842148065567, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.210278034210205, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.456613153219223, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.45845362544059753, "metrics/set_slot/angular_recall_all_dr0p10": 0.924604594707489, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7598460912704468, "metrics/set_slot/soft_count_bias_mean": 0.10197392106056213}, "step_train_time": 4.9084553718566895, "step_valid_time": 56372.15785121918, "step_total_time": 56377.06630706787, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0110108871309351, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19221547493093816, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3010, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2816, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2752, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9302940204823258, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935548172757475, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9329136988570482, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06445182724252492, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9091509745622729, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9142857142857143, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9117111147921153, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9302940204823258, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9886371692461531, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18818816819721518, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2090, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2083, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9239610963748894, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9359605911330049, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9299221357063404, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06403940886699508, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.920866489832007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9328257948947604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9268075639599556, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9239610963748894, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9975048697887683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19083000936125205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1271, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1150, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9021739130434783, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9142407553107789, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9081672528331379, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08575924468922108, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8928571428571429, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9047993705743509, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8987885892926925, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9021739130434783} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_23000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..031353b1678bb518fe8ef02841bd65eab6db56fa --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.089634261891226, "train": {"Classification_binary": 0.2732267379760742, "Cardinality": 0.19524303078651428, "Classification": 0.13961322605609894, "Regression_pt": 0.23795153200626373, "Regression_eta": 0.0027746667619794607, "Regression_sin_phi": 0.001471149967983365, "Regression_cos_phi": 0.0013137335190549493, "Regression_energy": 0.2358057200908661, "HitGrouping": 0.270823210477829, "Auxiliary": 0.34692052006721497, "Total": 1.7098923921585083, "Total_unweighted": 0.8921567663783208}, "train_group": {}, "valid": {"Classification_binary": 0.24785240491231283, "Cardinality": 0.17003814379374185, "Classification": 0.12795978784561157, "Regression_pt": 0.2358501156171163, "Regression_eta": 0.002604782891770204, "Regression_sin_phi": 0.0014448788327475388, "Regression_cos_phi": 0.0014865429451068242, "Regression_energy": 0.23297246297200522, "HitGrouping": 0.26591525475184125, "Auxiliary": 0.3240762750307719, "Total": 1.6277281443277996, "Total_unweighted": 0.8501709760166705, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.6663135886192322, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.28019091486930847, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.45470860600471497, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.26222679018974304, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.29489725828170776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.7174836993217468, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.5829289555549622, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.21672570705413818, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.43104031682014465, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.19453802704811096, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2698488235473633, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.8163334727287292, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7032756805419922, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2895386219024658, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4277450740337372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2753041088581085, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.28871971368789673, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6629480719566345, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.6805869340896606, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.27999958395957947, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.43671733140945435, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.263851523399353, "metrics/hit_grouping/calorimeter/all/object_purity": 0.28901493549346924, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6942552924156189, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02567940577864647, "metrics/hit_grouping/calorimeter/background_recall": 0.6280035376548767, "metrics/hit_grouping/calorimeter/completeness": 0.3191116750240326, "metrics/hit_grouping/calorimeter/energy_completeness": 0.36010342836380005, "metrics/hit_grouping/calorimeter/energy_purity": 0.3558647930622101, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5420636534690857, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.6977314352989197, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.2944009602069855, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3650044798851013, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2803366184234619, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3158557415008545, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.6524932980537415, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.6683745384216309, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2879365086555481, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4824114441871643, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.2699292004108429, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2970128357410431, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.7207666039466858, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.657024085521698, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24124550819396973, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5408924221992493, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.22262436151504517, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2238672524690628, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7550246715545654, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.6578961610794067, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2917789816856384, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5304889678955078, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.2732503414154053, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2958541214466095, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.7397072315216064, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6811591386795044, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4121849536895752, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.7153322696685791, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.38837292790412903, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3742315471172333, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.714809000492096, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7049078345298767, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.25042107701301575, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3002544641494751, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2378130406141281, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.27075788378715515, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.6427614688873291, "metrics/hit_grouping/calorimeter/purity": 0.31437620520591736, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8432618975639343, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05759109556674957, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16615384817123413, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05538644641637802, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02785724401473999, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7508137226104736, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12463582307100296, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11345496028661728, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1221792921423912, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.12640240788459778, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4315849542617798, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.673080563545227, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.29367589950561523, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.46107685565948486, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.27644315361976624, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.30393412709236145, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.723143994808197, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6870585083961487, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44157204031944275, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3850511312484741, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.40515562891960144, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6663179993629456, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7454822063446045, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34155917167663574, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4199213683605194, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.43565329909324646, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.537296712398529, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.628383219242096, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4462471306324005, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.33208563923835754, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3557667136192322, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7379695773124695, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6567906737327576, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43595314025878906, "metrics/hit_grouping/tracker/all/object_completeness": 0.3562239706516266, "metrics/hit_grouping/tracker/all/object_purity": 0.378105103969574, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6984224319458008, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006505885161459446, "metrics/hit_grouping/tracker/background_recall": 0.027720192447304726, "metrics/hit_grouping/tracker/completeness": 0.3722594082355499, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5270666480064392, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6669356822967529, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4224145710468292, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.41122180223464966, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.442636638879776, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6912984251976013, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6440278887748718, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4525033235549927, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.33005911111831665, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.34831002354621887, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7173044085502625, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6478303074836731, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4484223425388336, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2619362771511078, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2665112614631653, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6941747665405273, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6780562400817871, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.47375988960266113, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3751891255378723, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.39010560512542725, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6729870438575745, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6972108483314514, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4740346074104309, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.39831340312957764, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42564770579338074, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6424766778945923, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6219795942306519, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3800046741962433, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3241163492202759, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.35411176085472107, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7413954734802246, "metrics/hit_grouping/tracker/purity": 0.3554021120071411, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.89407879114151, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.011294526048004627, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.022776782512664795, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.022835683077573776, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07602085173130035, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6547882556915283, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.214618980884552, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.18243305385112762, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1772175282239914, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6132711172103882, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6247751116752625, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5188595652580261, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.42132270336151123, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.44925448298454285, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7926923036575317, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.367829293012619, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4504758417606354, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.48933419585227966, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5468734502792358, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11789806932210922, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.01606868952512741, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.838363647460938, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2739327847957611, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1434929370880127, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.721646249294281, "metrics/particle/class_1/pid_accuracy": 0.9469456076622009, "metrics/particle/class_2/efficiency": 0.3396903872489929, "metrics/particle/class_2/pid_accuracy": 0.43163803219795227, "metrics/particle/class_3/efficiency": 0.5869821906089783, "metrics/particle/class_3/pid_accuracy": 0.9302291870117188, "metrics/particle/class_4/efficiency": 0.5832531452178955, "metrics/particle/class_4/pid_accuracy": 0.2970297038555145, "metrics/particle/class_5/efficiency": 0.8339999914169312, "metrics/particle/class_5/pid_accuracy": 0.38129496574401855, "metrics/particle/count/bias_mean": -7.21875, "metrics/particle/count/mae": 7.79296875, "metrics/particle/count/prediction_mean": 62.83723831176758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09481730312108994, "metrics/particle/event/energy_response_mean": 0.9666719436645508, "metrics/particle/event/met_abs_error": 10.275980949401855, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07979024201631546, "metrics/particle/event/scalar_pt_response_mean": 0.9940271973609924, "metrics/particle/event/vector_pt_closure": 0.07890454679727554, "metrics/particle/matched/delta_eta_abs_mean": 0.018783453851938248, "metrics/particle/matched/delta_phi_abs_mean": 0.017817897722125053, "metrics/particle/matched/delta_r_mean": 0.029013589024543762, "metrics/particle/matched/energy_relative_abs_mean": 0.17395521700382233, "metrics/particle/matched/pid_accuracy": 0.9079459309577942, "metrics/particle/matched/pt_relative_abs_mean": 0.174833744764328, "metrics/particle/matching/duplicate_fraction": 0.07920802384614944, "metrics/particle/matching/efficiency": 0.630856990814209, "metrics/particle/matching/f1": 0.6651251316070557, "metrics/particle/matching/purity": 0.7033299207687378, "metrics/particle/matching/target_coverage_dr0p05": 0.5667155981063843, "metrics/particle/matching/target_coverage_dr0p10": 0.7066799402236938, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.42447447776794434, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6912533044815063, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027314918115735054, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17154017090797424, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17366482317447662, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02558743767440319, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2148679792881012, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01179985050112009, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9904255867004395, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.09732755273580551, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.454213857650757, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6695.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3997393846511841, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5477992296218872, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.548324704170227, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3683624267578125, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.22282783687114716, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.667630195617676, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025969862937927246, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5525085926055908, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.10562177002429962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.086832046508789, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9357.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6121982336044312, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0229928493499756, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.483997344970703, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.624557018280029, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1598581075668335, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.31100016832351685, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.730371475219727, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.5760879516601562, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.16802215576172, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02752334624528885, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17581376433372498, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1745811104774475, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02975030429661274, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21499216556549072, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030573248863220215, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7398381233215332, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1311257779598236, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.25813627243042, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1570.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4814828038215637, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0810407400131226, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1785550117492676, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.0159075260162354, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22398190200328827, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.083168029785156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06787330657243729, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8280065059661865, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.18908393383026123, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.5189008712768555, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2210.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5196954607963562, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.591692924499512, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.489091873168945, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6432358026504517, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04800323024392128, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.13394445180892944, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.30361270904541, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.403801441192627, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.080625534057617, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.029830336570739746, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17386490106582642, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17451554536819458, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01597391441464424, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2150927037000656, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009211365249939263, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3312436640262604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010054027661681175, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.899082362651825, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14113.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.36048296093940735, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.0618550777435303, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.625012993812561, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.4274630546569824, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.22727952897548676, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.362138748168945, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008524851873517036, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6979917287826538, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.053322914987802505, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.573481798171997, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20059.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6583477258682251, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8918876051902771, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.579911231994629, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.62956714630127, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2045316696166992, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.938279390335083, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.599176406860352, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3266424834728241, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.293668746948242, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028915926814079285, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17330612242221832, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17426563799381256, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019816594198346138, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21501843631267548, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006256144493818283, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6272807121276855, "metrics/particle/momentum/all/accepted/pz_bias": -0.04465850815176964, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.893114686012268, "metrics/particle/momentum/all/accepted/samples": 22378.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3805529475212097, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.8412258625030518, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6410048007965088, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4510972499847412, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2257319986820221, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.254661560058594, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01783342845737934, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0996564626693726, "metrics/particle/momentum/all/assigned/pz_bias": -0.0782831460237503, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6183815002441406, "metrics/particle/momentum/all/assigned/samples": 31626.0, "metrics/particle/momentum/all/efficiency": 0.6322540640830994, "metrics/particle/momentum/all/event/energy_bias": -7.7227983474731445, "metrics/particle/momentum/all/event/p3_error_mean": 27.616924285888672, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.463582992553711, "metrics/particle/momentum/all/event/px_bias": 1.3163865804672241, "metrics/particle/momentum/all/event/py_bias": 1.3832240104675293, "metrics/particle/momentum/all/event/pz_abs_mean": 20.76633071899414, "metrics/particle/momentum/all/event/pz_bias": -6.30653190612793, "metrics/particle/momentum/all/event/pz_residual_rms": 32.36205291748047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03208359703421593, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17369578778743744, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17210038006305695, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004825739189982414, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21208372712135315, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006082025356590748, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.685359537601471, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0558239184319973, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.87592351436615, "metrics/particle/momentum/class_1/accepted/samples": 12167.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.24904657900333405, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.7826647162437439, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.8196860551834106, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.6313956379890442, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1661870926618576, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.316657543182373, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.012788541615009308, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9667559266090393, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06918193399906158, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8363454341888428, "metrics/particle/momentum/class_1/assigned/samples": 15639.0, "metrics/particle/momentum/class_1/efficiency": 0.7291304469108582, "metrics/particle/momentum/class_1/event/energy_bias": 18.70888900756836, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.674558639526367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.577488899230957, "metrics/particle/momentum/class_1/event/px_bias": -1.4087824821472168, "metrics/particle/momentum/class_1/event/py_bias": -2.568803310394287, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.716693878173828, "metrics/particle/momentum/class_1/event/pz_bias": -6.601762771606445, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.94477653503418, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03840937092900276, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.208525612950325, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20700837671756744, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04028196260333061, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24680697917938232, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030456852167844772, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8422998189926147, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09804674983024597, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.018726348876953, "metrics/particle/momentum/class_2/accepted/samples": 985.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7161984443664551, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7448025345802307, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8666672110557556, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.45503711700439453, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3471118211746216, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.190406560897827, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.062003180384635925, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7984352111816406, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0014619875000789762, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.213590621948242, "metrics/particle/momentum/class_2/assigned/samples": 1887.0, "metrics/particle/momentum/class_2/efficiency": 0.33087000250816345, "metrics/particle/momentum/class_2/event/energy_bias": -21.66504669189453, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.90921974182129, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.120192527770996, "metrics/particle/momentum/class_2/event/px_bias": 2.783632755279541, "metrics/particle/momentum/class_2/event/py_bias": 0.8822572827339172, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.41218376159668, "metrics/particle/momentum/class_2/event/pz_bias": 1.4934221506118774, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.310319900512695, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023576367646455765, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1684560924768448, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17334575951099396, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04790773615241051, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21494852006435394, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002814998384565115, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3719905912876129, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.033734031021595, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3946810960769653, "metrics/particle/momentum/class_3/accepted/samples": 8881.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4870372712612152, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.1370880603790283, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.637432098388672, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.476227283477783, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.27694568037986755, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.95307159423828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016446402296423912, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9845321178436279, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.09159314632415771, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.882045030593872, "metrics/particle/momentum/class_3/assigned/samples": 13620.0, "metrics/particle/momentum/class_3/efficiency": 0.5840073823928833, "metrics/particle/momentum/class_3/event/energy_bias": -0.7894546985626221, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.873544692993164, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.890823364257812, "metrics/particle/momentum/class_3/event/px_bias": 0.206484854221344, "metrics/particle/momentum/class_3/event/py_bias": 2.8009140491485596, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.220909118652344, "metrics/particle/momentum/class_3/event/pz_bias": -0.6542471051216125, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.43796157836914, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02906455099582672, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17684349417686462, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17321670055389404, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.016069933772087097, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21609634160995483, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.023364486172795296, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2509909868240356, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.16371490061283112, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.564483642578125, "metrics/particle/momentum/class_4/accepted/samples": 214.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.444814532995224, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.516874313354492, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.492844104766846, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.328621864318848, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2896341383457184, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 31.916078567504883, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03963414579629898, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6809440851211548, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.18503493070602417, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.406976699829102, "metrics/particle/momentum/class_4/assigned/samples": 328.0, "metrics/particle/momentum/class_4/efficiency": 0.5879120826721191, "metrics/particle/momentum/class_4/event/energy_bias": -2.6196277141571045, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0450053215026855, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.215867757797241, "metrics/particle/momentum/class_4/event/px_bias": -0.09908732771873474, "metrics/particle/momentum/class_4/event/py_bias": 0.13840579986572266, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7182180881500244, "metrics/particle/momentum/class_4/event/pz_bias": -0.1416412591934204, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.087583065032959, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.025073906406760216, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1953202784061432, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19325189292430878, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03426598757505417, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23138950765132904, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04580152779817581, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.3854360580444336, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6267757415771484, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.617703437805176, "metrics/particle/momentum/class_5/accepted/samples": 131.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06390666961669922, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3184249699115753, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3188696503639221, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08782334625720978, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1184210553765297, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9834141731262207, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06578947603702545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.745487689971924, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5453780293464661, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.982706546783447, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.8238993883132935, "metrics/particle/momentum/class_5/event/energy_bias": -1.3575589656829834, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.3005480766296387, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.468224287033081, "metrics/particle/momentum/class_5/event/px_bias": -0.1658611297607422, "metrics/particle/momentum/class_5/event/py_bias": 0.130450040102005, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.800110101699829, "metrics/particle/momentum/class_5/event/pz_bias": -0.40230295062065125, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.350902557373047, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5133198499679565, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5173914432525635, "metrics/set_slot/angular_recall_all_dr0p10": 0.9059903621673584, "metrics/set_slot/angular_recall_selected_dr0p10": 0.714198112487793, "metrics/set_slot/soft_count_bias_mean": -1.334454894065857}, "step_train_time": 5.405835390090942, "step_valid_time": 58886.1127948761, "step_total_time": 58891.518630981445, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.012191865790236, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21795993879086806, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2771, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2724, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9154278163197885, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9236666666666666, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.919528787124606, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07633333333333334, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8999008919722498, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.908, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9039323046291687, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9154278163197885, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009854786503756, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20832562620444617, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2227, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2053, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2038, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9076038903625111, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9218679838347553, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9146803296948095, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07813201616524472, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.900972590627763, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9151324651998204, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9079973267988416, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9076038903625111, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9899536014546352, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19494350261608306, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1270, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1151, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1137, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8936335403726708, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9062992125984252, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8999218139171228, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09370078740157481, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8827639751552795, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8952755905511811, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8889757623143081, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8936335403726708} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_24000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..ee6a2a85e04f019a66411e655cb1da314f9dea34 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.137009664582149, "train": {"Classification_binary": 0.22508728504180908, "Cardinality": 0.15966260433197021, "Classification": 0.11827214062213898, "Regression_pt": 0.20577000081539154, "Regression_eta": 0.0021619123872369528, "Regression_sin_phi": 0.0012406933819875121, "Regression_cos_phi": 0.0011979122646152973, "Regression_energy": 0.20228254795074463, "HitGrouping": 0.24472305178642273, "Auxiliary": 0.3029261827468872, "Total": 1.4637222290039062, "Total_unweighted": 0.756012492463924}, "train_group": {}, "valid": {"Classification_binary": 0.22807488838831583, "Cardinality": 0.15764160950978598, "Classification": 0.11431171496709187, "Regression_pt": 0.2085133989651998, "Regression_eta": 0.0023339497856795788, "Regression_sin_phi": 0.001226296027501424, "Regression_cos_phi": 0.0012223155548175175, "Regression_energy": 0.20522522926330566, "HitGrouping": 0.24635211626688638, "Auxiliary": 0.2977500756581624, "Total": 1.4682539304097493, "Total_unweighted": 0.7609077929519117, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7150800824165344, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.334105908870697, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4450741410255432, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3219853341579437, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.336064875125885, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6237838864326477, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6456764936447144, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.26568904519081116, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4140969216823578, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.25161629915237427, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3046581447124481, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7089122533798218, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7385461926460266, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.33060988783836365, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.42438796162605286, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3191942870616913, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3171229660511017, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5955479741096497, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7226648926734924, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3259117007255554, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4301215708255768, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3140309453010559, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3221002519130707, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6148024201393127, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025798505172133446, "metrics/hit_grouping/calorimeter/background_recall": 0.6298233866691589, "metrics/hit_grouping/calorimeter/completeness": 0.3640684187412262, "metrics/hit_grouping/calorimeter/energy_completeness": 0.39205312728881836, "metrics/hit_grouping/calorimeter/energy_purity": 0.3874920904636383, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5196443200111389, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7455914616584778, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3431704640388489, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3571215271949768, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3339855968952179, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3455095589160919, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5624365210533142, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7123738527297974, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.33526331186294556, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.47646772861480713, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.32057681679725647, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.33388736844062805, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6441047787666321, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.6848611235618591, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.279635488986969, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5363417863845825, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.26497215032577515, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2602672874927521, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6951080560684204, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.6974371075630188, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3413363993167877, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5161008834838867, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3269359767436981, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.33817076683044434, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.666209876537323, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7033333778381348, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4306502044200897, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6823652386665344, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4133872389793396, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.41623157262802124, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.646258533000946, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7522600889205933, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.29592660069465637, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3054104745388031, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.28744328022003174, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2927781343460083, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.555510938167572, "metrics/hit_grouping/calorimeter/purity": 0.3587096631526947, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8568967580795288, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06885236501693726, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14153845608234406, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06823809444904327, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.026917463168501854, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7739593386650085, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.13787372410297394, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11915621161460876, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13573215901851654, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13157333433628082, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.40763968229293823, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7168536186218262, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3422059416770935, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.45415791869163513, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.32952484488487244, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3393501937389374, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6388378739356995, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7170546650886536, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43215781450271606, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.41868528723716736, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4169142544269562, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6262701749801636, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7705976366996765, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3409983217716217, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4430141746997833, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4438382089138031, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4699943959712982, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6656888723373413, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43498465418815613, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3647264540195465, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3639116585254669, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6808695197105408, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6907593607902527, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4261760711669922, "metrics/hit_grouping/tracker/all/object_completeness": 0.3883746564388275, "metrics/hit_grouping/tracker/all/object_purity": 0.38739559054374695, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6458530426025391, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00409019785001874, "metrics/hit_grouping/tracker/background_recall": 0.01622934266924858, "metrics/hit_grouping/tracker/completeness": 0.40747135877609253, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5206857919692993, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7027023434638977, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41822323203086853, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.44819581508636475, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4493778944015503, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6390888094902039, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6819071173667908, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4456501007080078, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.36811432242393494, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3687102496623993, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6603845357894897, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.673653244972229, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42313915491104126, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2771063446998596, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2696566581726074, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6454287767410278, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7155543565750122, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4429367482662201, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4053138494491577, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.40317502617836, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6068475842475891, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7299432754516602, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4733688533306122, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42846959829330444, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42494845390319824, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.576564610004425, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6515680551528931, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3960430920124054, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3592570722103119, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.36023589968681335, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7088503837585449, "metrics/hit_grouping/tracker/purity": 0.38807591795921326, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.876048743724823, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0152041707187891, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03076653741300106, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02967030368745327, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08905299752950668, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6625193357467651, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19388283789157867, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.18343617022037506, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.18984386324882507, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5956454277038574, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6687972545623779, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5084705948829651, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4603031277656555, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.45849892497062683, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7272297739982605, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4069879651069641, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.46023255586624146, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.4974185526371002, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5607988238334656, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07873938977718353, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.029993992298841476, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.06777000427246, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27217578887939453, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1417359858751297, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7967352867126465, "metrics/particle/class_1/pid_accuracy": 0.9566394090652466, "metrics/particle/class_2/efficiency": 0.3561812937259674, "metrics/particle/class_2/pid_accuracy": 0.46960631012916565, "metrics/particle/class_3/efficiency": 0.6373758316040039, "metrics/particle/class_3/pid_accuracy": 0.9419387578964233, "metrics/particle/class_4/efficiency": 0.6496631503105164, "metrics/particle/class_4/pid_accuracy": 0.3437037169933319, "metrics/particle/class_5/efficiency": 0.8379999995231628, "metrics/particle/class_5/pid_accuracy": 0.35322195291519165, "metrics/particle/count/bias_mean": -4.147786617279053, "metrics/particle/count/mae": 5.948567867279053, "metrics/particle/count/prediction_mean": 65.908203125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09409897774457932, "metrics/particle/event/energy_response_mean": 0.9503859877586365, "metrics/particle/event/met_abs_error": 9.607367515563965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07570267468690872, "metrics/particle/event/scalar_pt_response_mean": 0.9704952836036682, "metrics/particle/event/vector_pt_closure": 0.07921239733695984, "metrics/particle/matched/delta_eta_abs_mean": 0.017951490357518196, "metrics/particle/matched/delta_phi_abs_mean": 0.016927095130085945, "metrics/particle/matched/delta_r_mean": 0.027599096298217773, "metrics/particle/matched/energy_relative_abs_mean": 0.17301101982593536, "metrics/particle/matched/pid_accuracy": 0.9209519624710083, "metrics/particle/matched/pt_relative_abs_mean": 0.17072638869285583, "metrics/particle/matching/duplicate_fraction": 0.07648540288209915, "metrics/particle/matching/efficiency": 0.6899800896644592, "metrics/particle/matching/f1": 0.7110289931297302, "metrics/particle/matching/purity": 0.7334024906158447, "metrics/particle/matching/target_coverage_dr0p05": 0.6240730285644531, "metrics/particle/matching/target_coverage_dr0p10": 0.7582941651344299, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4684497117996216, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7447168231010437, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026600942015647888, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17016804218292236, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16751687228679657, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024316027760505676, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20874494314193726, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0129128722473979, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9420710206031799, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.20230832695960999, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.5184166431427, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7357.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3602948784828186, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2737114429473877, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5131241083145142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3304193019866943, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.19173316657543182, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.661038398742676, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.024554941803216934, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.402632474899292, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.2939920723438263, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7690587043762207, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9774.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6727322340011597, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.6846578121185303, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.740453720092773, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.822201728820801, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.09730064868927, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.15766045451164246, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.036839485168457, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -8.543978691101074, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.632293701171875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026352623477578163, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18131192028522491, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17497405409812927, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012601262889802456, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2168252170085907, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030667468905448914, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6879628896713257, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.24011798202991486, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.017081260681152, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1663.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5311676263809204, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2333321571350098, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.900918483734131, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.7081828117370605, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22827938199043274, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.999547958374023, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06260647624731064, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7437100410461426, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5850657224655151, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.600611209869385, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2348.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5504799485206604, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.213375091552734, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.526798248291016, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.817172884941101, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10664217919111252, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.16843606531620026, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.329446792602539, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -6.486297130584717, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.331417083740234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028179386630654335, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17292411625385284, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17103524506092072, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.028546785935759544, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21212810277938843, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008431703317910433, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.30552563071250916, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.038712240755558014, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8653708100318909, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15418.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3132411539554596, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5461761951446533, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.192164421081543, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0009143352508545, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.19432884454727173, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.708949089050293, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0058066388592123985, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5731579065322876, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.1149853989481926, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9517648220062256, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20666.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7192237973213196, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.361440658569336, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.999052047729492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.977807998657227, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.4921376705169678, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.06359580159187317, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.618380069732666, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.478747844696045, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.081764221191406, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027579888701438904, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17266520857810974, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17024408280849457, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.026188038289546967, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21143978834152222, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006506260950118303, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.591230571269989, "metrics/particle/momentum/all/accepted/pz_bias": -0.10166805982589722, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8654617071151733, "metrics/particle/momentum/all/accepted/samples": 24438.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.34287378191947937, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4425519704818726, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4102081060409546, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.221398949623108, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1959863305091858, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.999062538146973, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015462974086403847, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9758582711219788, "metrics/particle/momentum/all/assigned/pz_bias": -0.202009916305542, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2821757793426514, "metrics/particle/momentum/all/assigned/samples": 32788.0, "metrics/particle/momentum/all/efficiency": 0.6904560327529907, "metrics/particle/momentum/all/event/energy_bias": -13.259474754333496, "metrics/particle/momentum/all/event/p3_error_mean": 30.591598510742188, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.222193717956543, "metrics/particle/momentum/all/event/px_bias": -3.6960806846618652, "metrics/particle/momentum/all/event/py_bias": -0.2625007629394531, "metrics/particle/momentum/all/event/pz_abs_mean": 24.421709060668945, "metrics/particle/momentum/all/event/pz_bias": -17.509023666381836, "metrics/particle/momentum/all/event/pz_residual_rms": 35.87367630004883, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029910841956734657, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1621699035167694, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16562823951244354, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04270080476999283, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20635145902633667, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006447743158787489, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6275410652160645, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.132304385304451, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8412449359893799, "metrics/particle/momentum/class_1/accepted/samples": 13338.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1812433898448944, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.48557838797569275, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5275229215621948, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.379719078540802, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.12954235076904297, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.3004486560821533, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010790028609335423, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8178243637084961, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.17210789024829865, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.379739999771118, "metrics/particle/momentum/class_1/assigned/samples": 16126.0, "metrics/particle/momentum/class_1/efficiency": 0.7993048429489136, "metrics/particle/momentum/class_1/event/energy_bias": 12.876527786254883, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.07850456237793, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.101183891296387, "metrics/particle/momentum/class_1/event/px_bias": -1.3475978374481201, "metrics/particle/momentum/class_1/event/py_bias": -2.38944673538208, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.232168197631836, "metrics/particle/momentum/class_1/event/pz_bias": -12.429835319519043, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.672395706176758, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036124374717473984, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21932972967624664, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21895545721054077, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06591641157865524, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2580680847167969, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03782735392451286, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9881846904754639, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08643610775470734, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.499727725982666, "metrics/particle/momentum/class_2/accepted/samples": 1031.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8396168351173401, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8089941740036011, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.006391167640686, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4891180098056793, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37266185879707336, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.274105072021484, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06378896534442902, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9547171592712402, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.39381739497184753, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.319899082183838, "metrics/particle/momentum/class_2/assigned/samples": 2085.0, "metrics/particle/momentum/class_2/efficiency": 0.34632179141044617, "metrics/particle/momentum/class_2/event/energy_bias": -25.681331634521484, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.382326126098633, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.87674331665039, "metrics/particle/momentum/class_2/event/px_bias": 0.3110104501247406, "metrics/particle/momentum/class_2/event/py_bias": 0.7438567876815796, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.797006607055664, "metrics/particle/momentum/class_2/event/pz_bias": -3.395362615585327, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.25179100036621, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023464590311050415, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18116676807403564, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1705465167760849, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013462923467159271, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21192872524261475, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0025797132402658463, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3564457893371582, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.048066139221191406, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2561982870101929, "metrics/particle/momentum/class_3/accepted/samples": 9691.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4567297697067261, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.6083858013153076, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.467785596847534, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.280543327331543, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24597032368183136, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.641836166381836, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012781367637217045, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8347127437591553, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.19617542624473572, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.1839029788970947, "metrics/particle/momentum/class_3/assigned/samples": 14083.0, "metrics/particle/momentum/class_3/efficiency": 0.637272298336029, "metrics/particle/momentum/class_3/event/energy_bias": 3.3806345462799072, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.869363784790039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.067018508911133, "metrics/particle/momentum/class_3/event/px_bias": -2.3748364448547363, "metrics/particle/momentum/class_3/event/py_bias": 1.3559467792510986, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.59431266784668, "metrics/particle/momentum/class_3/event/pz_bias": -1.2137629985809326, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.983447074890137, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.030600273981690407, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.196451336145401, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1884692907333374, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0625336617231369, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2300727814435959, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008196720853447914, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.14515221118927, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.22867539525032043, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2063770294189453, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3540303111076355, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.807880163192749, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3285529613494873, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.189763069152832, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2316715568304062, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.259017944335938, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014662756584584713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2184087038040161, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.3005894124507904, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.489551067352295, "metrics/particle/momentum/class_4/assigned/samples": 341.0, "metrics/particle/momentum/class_4/efficiency": 0.6703296899795532, "metrics/particle/momentum/class_4/event/energy_bias": -1.4434845447540283, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.3254287242889404, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2906925678253174, "metrics/particle/momentum/class_4/event/px_bias": -0.08128984272480011, "metrics/particle/momentum/class_4/event/py_bias": -0.12876203656196594, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.97671377658844, "metrics/particle/momentum/class_4/event/pz_bias": -0.019345730543136597, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.448951721191406, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021943815052509308, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20014584064483643, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19984732568264008, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.05468427762389183, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2406633496284485, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05223880708217621, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.1999857425689697, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.8146753907203674, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.292421340942383, "metrics/particle/momentum/class_5/accepted/samples": 134.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10433554649353027, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5870504379272461, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5547639727592468, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2644914984703064, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1111111119389534, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.6589295864105225, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09803921729326248, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.116905927658081, "metrics/particle/momentum/class_5/assigned/pz_bias": -1.0571281909942627, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.884494304656982, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.8427672982215881, "metrics/particle/momentum/class_5/event/energy_bias": -2.3918190002441406, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.110100746154785, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.389456272125244, "metrics/particle/momentum/class_5/event/px_bias": -0.20336668193340302, "metrics/particle/momentum/class_5/event/py_bias": 0.15590429306030273, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.618255615234375, "metrics/particle/momentum/class_5/event/pz_bias": -0.450716495513916, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.952755928039551, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4714381694793701, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.47460809350013733, "metrics/set_slot/angular_recall_all_dr0p10": 0.9231734275817871, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7535360455513, "metrics/set_slot/soft_count_bias_mean": 0.7271082401275635}, "step_train_time": 4.843393564224243, "step_valid_time": 61342.15139007568, "step_total_time": 61346.99478435516, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010867125794649, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20591798650346904, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2788, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2745, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9210439378923027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9231788079470199, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9221101372581445, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07682119205298013, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9068384539147671, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9089403973509934, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9078882090292707, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9210439378923027, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9848184075727214, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2059447688420225, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2235, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2066, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2046, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9133510167992926, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9243847874720358, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9188347787413832, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0756152125279642, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9045092838196287, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9154362416107382, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9099399599733156, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9133510167992926, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9834016566225463, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1954514198764199, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1152, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1138, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8944099378881988, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9128367670364501, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9035294117647059, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08716323296354993, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8835403726708074, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.901743264659271, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8925490196078432, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8944099378881988} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_25000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..24937c40021fc4ca5edd94c323b7b395fcfa4d4a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1843850672730718, "train": {"Classification_binary": 0.2273608148097992, "Cardinality": 0.1879996657371521, "Classification": 0.1117350310087204, "Regression_pt": 0.2691051661968231, "Regression_eta": 0.002414275426417589, "Regression_sin_phi": 0.00136384810321033, "Regression_cos_phi": 0.0014049466699361801, "Regression_energy": 0.26625391840934753, "HitGrouping": 0.25374212861061096, "Auxiliary": 0.33295392990112305, "Total": 1.6260874271392822, "Total_unweighted": 0.8796380006242543}, "train_group": {}, "valid": {"Classification_binary": 0.2418620785077413, "Cardinality": 0.15592257181803384, "Classification": 0.11515769362449646, "Regression_pt": 0.26333846648534137, "Regression_eta": 0.002601785119622946, "Regression_sin_phi": 0.0014172640318671863, "Regression_cos_phi": 0.0013964885535339515, "Regression_energy": 0.26171253124872845, "HitGrouping": 0.2584156592686971, "Auxiliary": 0.33588961760203045, "Total": 1.6228952407836914, "Total_unweighted": 0.8874863075713316, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.6963261961936951, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.29742440581321716, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.45962029695510864, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.2840551733970642, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.28891870379447937, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6753565669059753, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6258479356765747, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2477187216281891, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42426297068595886, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.23064133524894714, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2837192118167877, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7583869695663452, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.722261905670166, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.29413455724716187, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.43430429697036743, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2838628888130188, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.27960848808288574, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6256068348884583, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7052679657936096, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2910299003124237, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4415501654148102, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2791510224342346, "metrics/hit_grouping/calorimeter/all/object_purity": 0.28297314047813416, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6535258293151855, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02539762482047081, "metrics/hit_grouping/calorimeter/background_recall": 0.6258628964424133, "metrics/hit_grouping/calorimeter/completeness": 0.3292408883571625, "metrics/hit_grouping/calorimeter/energy_completeness": 0.35765236616134644, "metrics/hit_grouping/calorimeter/energy_purity": 0.35342487692832947, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5488637089729309, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7337585687637329, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.31132397055625916, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3750373125076294, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.30222874879837036, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3086836040019989, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5948641300201416, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.6904608011245728, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.29483288526535034, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4890829622745514, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.28098171949386597, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2866581678390503, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6861960291862488, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.660393238067627, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24411089718341827, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.532802402973175, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2283952683210373, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.22471246123313904, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.743648111820221, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.6757275462150574, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.29917117953300476, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5256929397583008, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.28474706411361694, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.28825411200523376, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.7095608711242676, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6777634024620056, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3887183368206024, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6797488331794739, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3701952397823334, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3999590575695038, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6985871195793152, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7405455112457275, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.26977697014808655, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3206106722354889, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2614845931529999, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2623259425163269, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5875183939933777, "metrics/hit_grouping/calorimeter/purity": 0.324261337518692, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8746232986450195, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0700879842042923, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.19846153259277344, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06975091993808746, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.027221189811825752, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7723738551139832, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1266203671693802, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12086658924818039, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.12538298964500427, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.12005815654993057, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4241733253002167, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.6977987289428711, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.30519524216651917, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4651757478713989, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.29245829582214355, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.29779449105262756, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6797940731048584, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7100969552993774, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43574416637420654, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3840023875236511, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3774661123752594, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.639420211315155, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7561817169189453, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32809871435165405, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.40481603145599365, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4158543348312378, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5215927958488464, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6606422662734985, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4271087646484375, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3332398533821106, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.33506637811660767, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6994565725326538, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6843196153640747, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4215000867843628, "metrics/hit_grouping/tracker/all/object_completeness": 0.3553125560283661, "metrics/hit_grouping/tracker/all/object_purity": 0.355271577835083, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.665501594543457, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0037729863543063402, "metrics/hit_grouping/tracker/background_recall": 0.01463010162115097, "metrics/hit_grouping/tracker/completeness": 0.37212756276130676, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5443214774131775, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6862627863883972, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4229612648487091, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4178176522254944, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4299911558628082, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6732574105262756, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6756049394607544, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42889776825904846, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.32801100611686707, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.32479584217071533, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6740909814834595, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6892654895782471, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41039642691612244, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.24556773900985718, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2217848002910614, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6391585469245911, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7157081961631775, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4355679452419281, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3654008209705353, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.35484278202056885, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6203271150588989, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7474528551101685, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5006657838821411, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4050000011920929, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3476884365081787, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5692409873008728, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6323274374008179, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3892531096935272, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3334333300590515, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.35716354846954346, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7412784099578857, "metrics/hit_grouping/tracker/purity": 0.3543016314506531, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8986722826957703, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012163336388766766, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.017992649227380753, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.01766628585755825, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07428323477506638, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6693768501281738, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19595645368099213, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.17623268067836761, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1674790382385254, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.59305340051651, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6568942070007324, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5028038620948792, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4215397536754608, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4225296080112457, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7540876865386963, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.40287530422210693, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4313981533050537, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.46637091040611267, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5570825338363647, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08285204321146011, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.02627778798341751, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.062875747680664, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26560643315315247, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.13516660034656525, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.69792240858078, "metrics/particle/class_1/pid_accuracy": 0.9310520887374878, "metrics/particle/class_2/efficiency": 0.33004263043403625, "metrics/particle/class_2/pid_accuracy": 0.3749150335788727, "metrics/particle/class_3/efficiency": 0.6249973177909851, "metrics/particle/class_3/pid_accuracy": 0.9403425455093384, "metrics/particle/class_4/efficiency": 0.5813282132148743, "metrics/particle/class_4/pid_accuracy": 0.28642383217811584, "metrics/particle/class_5/efficiency": 0.6600000262260437, "metrics/particle/class_5/pid_accuracy": 0.3181818127632141, "metrics/particle/count/bias_mean": -4.331380367279053, "metrics/particle/count/mae": 5.840494632720947, "metrics/particle/count/prediction_mean": 65.724609375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09499979019165039, "metrics/particle/event/energy_response_mean": 0.9816791415214539, "metrics/particle/event/met_abs_error": 9.826781272888184, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07966750115156174, "metrics/particle/event/scalar_pt_response_mean": 1.0058387517929077, "metrics/particle/event/vector_pt_closure": 0.07851018756628036, "metrics/particle/matched/delta_eta_abs_mean": 0.019499454647302628, "metrics/particle/matched/delta_phi_abs_mean": 0.01871085911989212, "metrics/particle/matched/delta_r_mean": 0.030218103900551796, "metrics/particle/matched/energy_relative_abs_mean": 0.18611113727092743, "metrics/particle/matched/pid_accuracy": 0.9023900628089905, "metrics/particle/matched/pt_relative_abs_mean": 0.18414802849292755, "metrics/particle/matching/duplicate_fraction": 0.08412826061248779, "metrics/particle/matching/efficiency": 0.6345556974411011, "metrics/particle/matching/f1": 0.6547979116439819, "metrics/particle/matching/purity": 0.6763741374015808, "metrics/particle/matching/target_coverage_dr0p05": 0.5665297508239746, "metrics/particle/matching/target_coverage_dr0p10": 0.7296061515808105, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4058602750301361, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6964481472969055, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02811412699520588, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18360446393489838, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1833711564540863, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024427926167845726, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2248709797859192, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012732175178825855, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.077354073524475, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04010282829403877, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.694155216217041, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6676.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4347429871559143, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.8154010772705078, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.427666425704956, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.2409684658050537, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.25640764832496643, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 30.363426208496094, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.027461349964141846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.6542900800704956, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06733296066522598, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.291196823120117, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9832.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6104608774185181, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5004773139953613, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.352755546569824, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.536931991577148, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.026039142161607742, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.72959965467453, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.553218841552734, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.8198356628417969, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.840038299560547, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027179304510354996, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18419964611530304, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18928058445453644, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03321195766329765, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23130951821804047, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.035862069576978683, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.9525959491729736, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05780784413218498, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.409072399139404, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1450.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6330592036247253, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7482812404632568, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.949732542037964, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.7774527072906494, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2884448170661926, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.92389488220215, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07124239951372147, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.276440382003784, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0447411835193634, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.567777633666992, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2302.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.47997352480888367, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.748420238494873, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.16292667388916, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.652217984199524, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11455480009317398, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11603580415248871, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.955448150634766, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.8991799354553223, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.915817260742188, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03125375136733055, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18760356307029724, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18434929847717285, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012499826960265636, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2247997373342514, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0015355622163042426, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.35670268535614014, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.043109502643346786, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9968739151954651, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14327.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3498193025588989, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.158548355102539, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.7627513408660889, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.5571409463882446, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.23838138580322266, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.2412166595459, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008577659726142883, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7203648090362549, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07068894803524017, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.34956431388855, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20635.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6683304309844971, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.1338934898376465, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.805414199829102, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.672816276550293, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.7787743806838989, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.5324152708053589, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.892932891845703, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.2716307640075684, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.617359161376953, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.030057117342948914, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18619467318058014, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18437692523002625, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017384009435772896, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22524695098400116, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007081459276378155, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6740373373031616, "metrics/particle/momentum/all/accepted/pz_bias": -0.03569834679365158, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0118894577026367, "metrics/particle/momentum/all/accepted/samples": 22453.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.39519721269607544, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.0267696380615234, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.1158862113952637, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.9182919263839722, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2473069131374359, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.450231552124023, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018645672127604485, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1801420450210571, "metrics/particle/momentum/all/assigned/pz_bias": -0.061573129147291183, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.7627267837524414, "metrics/particle/momentum/all/assigned/samples": 32769.0, "metrics/particle/momentum/all/efficiency": 0.6343730688095093, "metrics/particle/momentum/all/event/energy_bias": -3.114048480987549, "metrics/particle/momentum/all/event/p3_error_mean": 27.746692657470703, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.440242767333984, "metrics/particle/momentum/all/event/px_bias": -0.9193683862686157, "metrics/particle/momentum/all/event/py_bias": -2.1459789276123047, "metrics/particle/momentum/all/event/pz_abs_mean": 20.83466148376465, "metrics/particle/momentum/all/event/pz_bias": -5.9906463623046875, "metrics/particle/momentum/all/event/pz_residual_rms": 31.717023849487305, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03530796989798546, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1971139758825302, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1949879676103592, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.030502744019031525, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2349223643541336, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007952796295285225, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7861409783363342, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05320822820067406, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.110091209411621, "metrics/particle/momentum/class_1/accepted/samples": 11694.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2501641809940338, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8313446044921875, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.88898104429245, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.7061086893081665, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.21188516914844513, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.744713306427002, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.015201959758996964, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1076563596725464, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0757758766412735, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1013338565826416, "metrics/particle/momentum/class_1/assigned/samples": 15919.0, "metrics/particle/momentum/class_1/efficiency": 0.7007850408554077, "metrics/particle/momentum/class_1/event/energy_bias": 21.196495056152344, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.455869674682617, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.79828929901123, "metrics/particle/momentum/class_1/event/px_bias": 0.10783964395523071, "metrics/particle/momentum/class_1/event/py_bias": -3.779198408126831, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.305030822753906, "metrics/particle/momentum/class_1/event/pz_bias": -5.071079254150391, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.304466247558594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04026327282190323, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21331767737865448, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21406564116477966, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03518744930624962, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2526017427444458, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03781512752175331, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0597455501556396, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.006721420679241419, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.570586681365967, "metrics/particle/momentum/class_2/accepted/samples": 952.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8604979515075684, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9638512134552002, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.237919807434082, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.793201744556427, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39200761914253235, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.238276958465576, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06945765763521194, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.163172960281372, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2224424183368683, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.34760046005249, "metrics/particle/momentum/class_2/assigned/samples": 2102.0, "metrics/particle/momentum/class_2/efficiency": 0.3197850286960602, "metrics/particle/momentum/class_2/event/energy_bias": -22.77518653869629, "metrics/particle/momentum/class_2/event/p3_error_mean": 24.078094482421875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.66568374633789, "metrics/particle/momentum/class_2/event/px_bias": 0.45453929901123047, "metrics/particle/momentum/class_2/event/py_bias": -0.14202134311199188, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.179948806762695, "metrics/particle/momentum/class_2/event/pz_bias": -1.2341818809509277, "metrics/particle/momentum/class_2/event/pz_residual_rms": 28.65638542175293, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02246539480984211, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16932493448257446, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16784587502479553, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.007142036687582731, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20905852317810059, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002321409760043025, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3610265254974365, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.005965793039649725, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2778308391571045, "metrics/particle/momentum/class_3/accepted/samples": 9477.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.48849543929100037, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.4798662662506104, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.5843966007232666, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.407677173614502, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.26457178592681885, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 33.26827621459961, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01423861924558878, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9316322207450867, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.083687424659729, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.5945091247558594, "metrics/particle/momentum/class_3/assigned/samples": 14257.0, "metrics/particle/momentum/class_3/efficiency": 0.6231998205184937, "metrics/particle/momentum/class_3/event/energy_bias": 2.3155670166015625, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.381080627441406, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.986777305603027, "metrics/particle/momentum/class_3/event/px_bias": -1.0990179777145386, "metrics/particle/momentum/class_3/event/py_bias": 1.0157033205032349, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.347498893737793, "metrics/particle/momentum/class_3/event/pz_bias": 0.07424240559339523, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.78141212463379, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03418977931141853, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19689655303955078, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18813735246658325, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.01493102964013815, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22797420620918274, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.018433179706335068, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.375710368156433, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.13239257037639618, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.696510076522827, "metrics/particle/momentum/class_4/accepted/samples": 217.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5159444808959961, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.315876007080078, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.133838653564453, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.951854944229126, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.302325576543808, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.14196014404297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.023255813866853714, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.8165333271026611, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.2076423615217209, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.771022796630859, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.5961538553237915, "metrics/particle/momentum/class_4/event/energy_bias": -2.121574878692627, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.020167589187622, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1588854789733887, "metrics/particle/momentum/class_4/event/px_bias": -0.0016790777444839478, "metrics/particle/momentum/class_4/event/py_bias": 0.19803856313228607, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.729445219039917, "metrics/particle/momentum/class_4/event/pz_bias": 0.3045910894870758, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.873071193695068, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.029439428821206093, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22195254266262054, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21535080671310425, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0336507223546505, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25909221172332764, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03539822995662689, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.302398443222046, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.888935387134552, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.07424259185791, "metrics/particle/momentum/class_5/accepted/samples": 113.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11647912114858627, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.39390864968299866, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3870510458946228, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0677221417427063, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.21088434755802155, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7860901355743408, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08163265138864517, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.2866268157958984, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.09814315289258957, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.01292085647583, "metrics/particle/momentum/class_5/assigned/samples": 147.0, "metrics/particle/momentum/class_5/efficiency": 0.7106918096542358, "metrics/particle/momentum/class_5/event/energy_bias": -1.7293497323989868, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.4323410987854004, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.6133580207824707, "metrics/particle/momentum/class_5/event/px_bias": -0.3810502290725708, "metrics/particle/momentum/class_5/event/py_bias": 0.5614989399909973, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7935024499893188, "metrics/particle/momentum/class_5/event/pz_bias": -0.06421878188848495, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.531129360198975, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5355647802352905, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5373021960258484, "metrics/set_slot/angular_recall_all_dr0p10": 0.9116777777671814, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7331933379173279, "metrics/set_slot/soft_count_bias_mean": -0.0023560028057545424}, "step_train_time": 4.938490152359009, "step_valid_time": 63788.138030052185, "step_total_time": 63793.07652044296, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211896294593885, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21580443351225476, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3006, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2749, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2675, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9081598942847704, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9145043246839654, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9113210674622907, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0854956753160346, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8837132474397092, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8898868928809048, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8867893253770927, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9081598942847704, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025006310226827, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20567558764966354, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2043, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2024, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.903183023872679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9149126735333631, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9090100111234706, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08508732646663682, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8947833775419982, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9064039408866995, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9005561735261401, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.903183023872679, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0111695198142565, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19580987857205567, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1134, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1119, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8804347826086957, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8985736925515055, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8894117647058823, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10142630744849446, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8687888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8866877971473851, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8776470588235294, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8804347826086957} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_26000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..a96b0c5759b37d745f9215c086cf6499312063f3 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2317604699639948, "train": {"Classification_binary": 0.21488036215305328, "Cardinality": 0.13591480255126953, "Classification": 0.12173322588205338, "Regression_pt": 0.24268297851085663, "Regression_eta": 0.002789632184430957, "Regression_sin_phi": 0.0014518225798383355, "Regression_cos_phi": 0.001294652814976871, "Regression_energy": 0.24368378520011902, "HitGrouping": 0.2579907774925232, "Auxiliary": 0.3141970634460449, "Total": 1.539846658706665, "Total_unweighted": 0.8285164593253285}, "train_group": {}, "valid": {"Classification_binary": 0.24236416816711426, "Cardinality": 0.17068145672480264, "Classification": 0.11412011583646138, "Regression_pt": 0.24009491999944052, "Regression_eta": 0.002531754473845164, "Regression_sin_phi": 0.001371594611555338, "Regression_cos_phi": 0.001322807278484106, "Regression_energy": 0.2389448881149292, "HitGrouping": 0.25810853640238446, "Auxiliary": 0.3286546270052592, "Total": 1.5958164532979329, "Total_unweighted": 0.84075024848183, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7023640871047974, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.28972944617271423, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4434683918952942, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.27665165066719055, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.27934566140174866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.667327880859375, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6384766697883606, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.23585395514965057, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.397492378950119, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2208898961544037, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.25881463289260864, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7434768080711365, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7216019034385681, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.29349619150161743, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4167957901954651, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.28213196992874146, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2745441198348999, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6304617524147034, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7079553008079529, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.28711381554603577, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.423647403717041, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2748752534389496, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2746783494949341, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6524620056152344, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025707632303237915, "metrics/hit_grouping/calorimeter/background_recall": 0.6287869811058044, "metrics/hit_grouping/calorimeter/completeness": 0.33374109864234924, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3666110634803772, "metrics/hit_grouping/calorimeter/energy_purity": 0.3623286783695221, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5502603650093079, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7321593761444092, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3009563982486725, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.36422812938690186, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2915083169937134, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.296442449092865, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5974918007850647, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.69413161277771, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2951769232749939, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.46385249495506287, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.27967312932014465, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2814546823501587, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6853469014167786, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.671128511428833, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24941106140613556, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5075211524963379, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.23466868698596954, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.22154931724071503, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7345468401908875, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.6851882934570312, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.30550792813301086, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5068203210830688, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.2908642292022705, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.28748682141304016, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.7005917429924011, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6994383335113525, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.41612935066223145, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6776556968688965, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.39908739924430847, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3776898980140686, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6797488331794739, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7335209846496582, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2508297562599182, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3017275929450989, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2417915314435959, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.24772673845291138, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5972278118133545, "metrics/hit_grouping/calorimeter/purity": 0.3287980556488037, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8482344150543213, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05986349284648895, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.18153846263885498, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05867032706737518, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02107488550245762, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05538461357355118, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7650049924850464, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.11191022396087646, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11003420501947403, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1093902513384819, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.10888592898845673, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4133409261703491, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7016840577125549, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3020344376564026, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.44684550166130066, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2890014946460724, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.28961941599845886, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6789414882659912, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7124301195144653, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4448595345020294, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3869711756706238, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.36650288105010986, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6404662132263184, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7620945572853088, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3381940424442291, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.40337345004081726, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3854066729545593, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.49186763167381287, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6610804200172424, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4315980076789856, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.33621200919151306, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.33021849393844604, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.702528178691864, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6858178377151489, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42792367935180664, "metrics/hit_grouping/tracker/all/object_completeness": 0.3579118847846985, "metrics/hit_grouping/tracker/all/object_purity": 0.34633490443229675, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6651709675788879, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0053712441585958, "metrics/hit_grouping/tracker/background_recall": 0.021323224529623985, "metrics/hit_grouping/tracker/completeness": 0.37494131922721863, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5618443489074707, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6923293471336365, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42605921626091003, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4153965413570404, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4120672941207886, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6656947731971741, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.67854243516922, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.443746417760849, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.33428531885147095, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.31544235348701477, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6708547472953796, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6790933609008789, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4152508080005646, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2554069757461548, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2332412749528885, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6579692363739014, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7145399451255798, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4462616741657257, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3738486170768738, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.35100504755973816, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6226635575294495, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.729098916053772, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4913448691368103, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40161851048469543, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3570704460144043, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5998668670654297, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6407900452613831, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39288222789764404, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.32946518063545227, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3383631706237793, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7320299744606018, "metrics/hit_grouping/tracker/purity": 0.35755422711372375, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8889633417129517, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012597741559147835, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.013073223643004894, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.010621951892971992, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07906168699264526, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.676307737827301, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.17522032558918, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.16263282299041748, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1639144867658615, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5785381197929382, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6592966318130493, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5131338238716125, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.42700526118278503, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4127241373062134, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7546780109405518, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3688060939311981, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.44083043932914734, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.4765356481075287, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5520339012145996, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11692126840353012, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.021229146048426628, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.92801856994629, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27838554978370667, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14794571697711945, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7528690099716187, "metrics/particle/class_1/pid_accuracy": 0.9362155199050903, "metrics/particle/class_2/efficiency": 0.3492259383201599, "metrics/particle/class_2/pid_accuracy": 0.39832958579063416, "metrics/particle/class_3/efficiency": 0.6418595910072327, "metrics/particle/class_3/pid_accuracy": 0.9427120089530945, "metrics/particle/class_4/efficiency": 0.6294513940811157, "metrics/particle/class_4/pid_accuracy": 0.26605504751205444, "metrics/particle/class_5/efficiency": 0.7559999823570251, "metrics/particle/class_5/pid_accuracy": 0.32804232835769653, "metrics/particle/count/bias_mean": -2.9290363788604736, "metrics/particle/count/mae": 5.361328125, "metrics/particle/count/prediction_mean": 67.126953125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.0909397229552269, "metrics/particle/event/energy_response_mean": 0.9720036387443542, "metrics/particle/event/met_abs_error": 9.80932331085205, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07465115934610367, "metrics/particle/event/scalar_pt_response_mean": 0.9925879836082458, "metrics/particle/event/vector_pt_closure": 0.0785728171467781, "metrics/particle/matched/delta_eta_abs_mean": 0.02082962542772293, "metrics/particle/matched/delta_phi_abs_mean": 0.018316568806767464, "metrics/particle/matched/delta_r_mean": 0.031007101759314537, "metrics/particle/matched/energy_relative_abs_mean": 0.181192085146904, "metrics/particle/matched/pid_accuracy": 0.906425952911377, "metrics/particle/matched/pt_relative_abs_mean": 0.1788051873445511, "metrics/particle/matching/duplicate_fraction": 0.08675453811883926, "metrics/particle/matching/efficiency": 0.670167088508606, "metrics/particle/matching/f1": 0.6844760179519653, "metrics/particle/matching/purity": 0.6994093656539917, "metrics/particle/matching/target_coverage_dr0p05": 0.5821422338485718, "metrics/particle/matching/target_coverage_dr0p10": 0.7444101572036743, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4401613175868988, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7319015860557556, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02902800403535366, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17986932396888733, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1787145584821701, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0025248436722904444, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21978706121444702, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012624113820493221, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.009656310081482, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0701519101858139, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.548746109008789, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7050.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4354819357395172, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7838162183761597, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.185060977935791, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.972169041633606, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.23022301495075226, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.70171356201172, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025202520191669464, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5896302461624146, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05317050218582153, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.604543209075928, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9999.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6446598172187805, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.2614954710006714, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.367944717407227, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.109445571899414, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.7832829356193542, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6846432685852051, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.73795223236084, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.222550630569458, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.583596229553223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02916226163506508, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19069215655326843, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18847404420375824, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0008491019834764302, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22963044047355652, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.041613318026065826, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.965922236442566, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13566678762435913, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.538951396942139, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1562.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.651374340057373, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7031266689300537, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.349898099899292, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.149643898010254, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26947638392448425, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.213293075561523, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07279693335294724, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0158073902130127, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0700518861413002, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.976019859313965, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2349.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5170473456382751, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.967219352722168, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.812137603759766, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8359850645065308, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.28391021490097046, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1515520066022873, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.592318534851074, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.431318998336792, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.561485290527344, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03219841793179512, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18129241466522217, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17832787334918976, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.005747050512582064, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21876446902751923, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0013888889225199819, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.337358683347702, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.05189327150583267, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9357695579528809, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15120.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.335122674703598, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.141347646713257, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.5993658304214478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.378836750984192, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2099410444498062, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.828987121582031, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008819441311061382, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6899353265762329, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0638514906167984, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.602847099304199, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20863.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7053225636482239, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.142829179763794, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.666979789733887, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.820709228515625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9501738548278809, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.6462535858154297, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.19871187210083, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.6354988813400269, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.613882541656494, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.031056758016347885, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18148833513259888, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1791105419397354, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.004467463586479425, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2197994589805603, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00737400958314538, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6442655324935913, "metrics/particle/momentum/all/accepted/pz_bias": -0.06283114105463028, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9605287313461304, "metrics/particle/momentum/all/accepted/samples": 23732.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3877067267894745, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.0027084350585938, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.8995182514190674, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.682722806930542, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.22025835514068604, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.054261207580566, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01827707700431347, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1253188848495483, "metrics/particle/momentum/all/assigned/pz_bias": -0.06107426807284355, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.752504587173462, "metrics/particle/momentum/all/assigned/samples": 33211.0, "metrics/particle/momentum/all/efficiency": 0.6705090999603271, "metrics/particle/momentum/all/event/energy_bias": -7.0858845710754395, "metrics/particle/momentum/all/event/p3_error_mean": 26.089345932006836, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.846875190734863, "metrics/particle/momentum/all/event/px_bias": -2.017366886138916, "metrics/particle/momentum/all/event/py_bias": -3.4824488162994385, "metrics/particle/momentum/all/event/pz_abs_mean": 19.5450439453125, "metrics/particle/momentum/all/event/pz_bias": -6.289368629455566, "metrics/particle/momentum/all/event/pz_residual_rms": 30.183637619018555, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03534006327390671, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1869698017835617, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18432900309562683, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.007551458664238453, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.22410368919372559, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00834724586457014, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7381174564361572, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.10580306500196457, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.0472702980041504, "metrics/particle/momentum/class_1/accepted/samples": 12579.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23924341797828674, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.7487381100654602, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7903900742530823, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5742030143737793, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.16583974659442902, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.713019847869873, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014016373082995415, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0234073400497437, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.10038593411445618, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.968754768371582, "metrics/particle/momentum/class_1/assigned/samples": 16124.0, "metrics/particle/momentum/class_1/efficiency": 0.7538203597068787, "metrics/particle/momentum/class_1/event/energy_bias": 9.928257942199707, "metrics/particle/momentum/class_1/event/p3_error_mean": 23.435178756713867, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.765023231506348, "metrics/particle/momentum/class_1/event/px_bias": 1.027000069618225, "metrics/particle/momentum/class_1/event/py_bias": -2.7692208290100098, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.99948501586914, "metrics/particle/momentum/class_1/event/pz_bias": -5.187539100646973, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.879013061523438, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04077688977122307, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21583117544651031, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21305154263973236, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04988373443484306, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25412696599960327, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03714565187692642, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9888064861297607, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13010306656360626, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.4378252029418945, "metrics/particle/momentum/class_2/accepted/samples": 1023.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.914128303527832, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9317165613174438, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1711184978485107, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6966104507446289, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3903328776359558, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.937042713165283, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06155950576066971, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0034384727478027, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.31278979778289795, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.0835280418396, "metrics/particle/momentum/class_2/assigned/samples": 2193.0, "metrics/particle/momentum/class_2/efficiency": 0.34363454580307007, "metrics/particle/momentum/class_2/event/energy_bias": -20.91666030883789, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.78633689880371, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.81074333190918, "metrics/particle/momentum/class_2/event/px_bias": -1.9031447172164917, "metrics/particle/momentum/class_2/event/py_bias": -1.7054497003555298, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.487930297851562, "metrics/particle/momentum/class_2/event/pz_bias": -0.11899417638778687, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.109174728393555, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02453804202377796, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17041954398155212, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16845887899398804, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.003692712401971221, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20967204868793488, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0024572541005909443, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.35351037979125977, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03043278679251671, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2413709163665771, "metrics/particle/momentum/class_3/accepted/samples": 9767.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.47599542140960693, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.5438103675842285, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.2425239086151123, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0640170574188232, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2552097737789154, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.516809463500977, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.015490413643419743, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9193416237831116, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0819745659828186, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.740144729614258, "metrics/particle/momentum/class_3/assigned/samples": 14396.0, "metrics/particle/momentum/class_3/efficiency": 0.642270028591156, "metrics/particle/momentum/class_3/event/energy_bias": 8.59089183807373, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.348943710327148, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.465336799621582, "metrics/particle/momentum/class_3/event/px_bias": -1.4003385305404663, "metrics/particle/momentum/class_3/event/py_bias": 0.5194730758666992, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.996697425842285, "metrics/particle/momentum/class_3/event/pz_bias": -1.3726451396942139, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.558841705322266, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03206881880760193, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1884213387966156, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18736033141613007, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03936292976140976, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22844737768173218, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.012658228166401386, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1895536184310913, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1800125688314438, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.68723464012146, "metrics/particle/momentum/class_4/accepted/samples": 237.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.39846137166023254, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.8581528663635254, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.0329391956329346, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.8817625045776367, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2543352544307709, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.916769027709961, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.023121386766433716, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6838576793670654, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.44468045234680176, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.992415428161621, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.651098906993866, "metrics/particle/momentum/class_4/event/energy_bias": -2.2279324531555176, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.78304386138916, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7428250312805176, "metrics/particle/momentum/class_4/event/px_bias": 0.4513004720211029, "metrics/particle/momentum/class_4/event/py_bias": 0.1732785850763321, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1490955352783203, "metrics/particle/momentum/class_4/event/pz_bias": 0.5037465691566467, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.335565567016602, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02792196162045002, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2003890573978424, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1927211880683899, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0428311787545681, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2352364957332611, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0396825410425663, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8708139657974243, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3074091374874115, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5476508140563965, "metrics/particle/momentum/class_5/accepted/samples": 126.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.15516476333141327, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2923406660556793, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2869569957256317, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.051904868334531784, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15131579339504242, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.39351940155029297, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09868421405553818, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.0759642124176025, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4566856622695923, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.01701021194458, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.7924528121948242, "metrics/particle/momentum/class_5/event/energy_bias": -2.4604411125183105, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.2742438316345215, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.52121639251709, "metrics/particle/momentum/class_5/event/px_bias": -0.19218449294567108, "metrics/particle/momentum/class_5/event/py_bias": 0.299469918012619, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6774531602859497, "metrics/particle/momentum/class_5/event/pz_bias": -0.11393673717975616, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.028285503387451, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5162431597709656, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5173025131225586, "metrics/set_slot/angular_recall_all_dr0p10": 0.9127093553543091, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7411018013954163, "metrics/set_slot/soft_count_bias_mean": 1.787994384765625}, "step_train_time": 9.92001461982727, "step_valid_time": 66228.68436646461, "step_total_time": 66238.6043817997, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010579587447527, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2205010475450011, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2970, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2758, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2697, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9111331351172778, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9286195286195286, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9197932299483075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07138047138047138, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8909811694747275, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.908080808080808, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8994497248624312, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9111331351172778, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9973735856280274, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19850111702885997, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2213, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2039, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2029, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9014146772767463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9213737008585631, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9112849162011173, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07862629914143697, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8969938107869142, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9168549480343425, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9068156424581005, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9014146772767463, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064798076515253, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19453089027286685, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1263, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1157, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1141, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8982919254658385, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9160728424386382, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.907095256762054, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08392715756136183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8858695652173914, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9034045922406968, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8945511564092513, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8982919254658385} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_27000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..0865a23f299a6c76057758a208be17d98d87c68d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2791358726549176, "train": {"Classification_binary": 0.21258653700351715, "Cardinality": 0.14268749952316284, "Classification": 0.12316959351301193, "Regression_pt": 0.22132767736911774, "Regression_eta": 0.0021233162842690945, "Regression_sin_phi": 0.0013365155318751931, "Regression_cos_phi": 0.0011920318938791752, "Regression_energy": 0.21978598833084106, "HitGrouping": 0.2620433568954468, "Auxiliary": 0.3039745092391968, "Total": 1.4773012399673462, "Total_unweighted": 0.7815216599265113}, "train_group": {}, "valid": {"Classification_binary": 0.23024998108545938, "Cardinality": 0.15842205286026, "Classification": 0.11177249749501546, "Regression_pt": 0.23162198066711426, "Regression_eta": 0.002225023228675127, "Regression_sin_phi": 0.0011936573622127373, "Regression_cos_phi": 0.0012630048052718241, "Regression_energy": 0.22991953293482462, "HitGrouping": 0.25777031977971393, "Auxiliary": 0.317482054233551, "Total": 1.5212799708048503, "Total_unweighted": 0.8082456775785734, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.6941275000572205, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2871493995189667, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.44781336188316345, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.273364782333374, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2888655960559845, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6782847046852112, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.650658369064331, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.23114468157291412, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41240257024765015, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.21658161282539368, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.25715580582618713, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7177228331565857, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7201415300369263, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2952219843864441, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4309472143650055, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.28324878215789795, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.28052178025245667, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6396549940109253, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7055380940437317, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.28687676787376404, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4347112476825714, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.27408838272094727, "metrics/hit_grouping/calorimeter/all/object_purity": 0.281110942363739, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.659088134765625, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0259423665702343, "metrics/hit_grouping/calorimeter/background_recall": 0.6309893131256104, "metrics/hit_grouping/calorimeter/completeness": 0.32949963212013245, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3636799454689026, "metrics/hit_grouping/calorimeter/energy_purity": 0.3595675826072693, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5547848343849182, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7256796956062317, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.30346736311912537, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.36793074011802673, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.29330578446388245, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.308015912771225, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.6113466620445251, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.6966692805290222, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2968411147594452, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4904172718524933, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.28100287914276123, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.28696927428245544, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6831635236740112, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.6721466779708862, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24137629568576813, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5180128812789917, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2262059599161148, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.21805694699287415, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7350524663925171, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.684800386428833, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3015788197517395, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5255060791969299, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.28641971945762634, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.29223865270614624, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.7000311613082886, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.701945424079895, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4054793119430542, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6792255640029907, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3904041349887848, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3789246082305908, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6750392317771912, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.728291928768158, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.25589442253112793, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3058122396469116, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.24594727158546448, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.25663137435913086, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.6130306720733643, "metrics/hit_grouping/calorimeter/purity": 0.3246976435184479, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8689896464347839, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06740889698266983, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14307692646980286, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06776190549135208, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02288355864584446, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7500734925270081, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12517133355140686, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11402508616447449, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1234084814786911, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1172579750418663, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4401368200778961, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.6994926929473877, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.30085521936416626, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4593717157840729, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2871525287628174, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.29603898525238037, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6847127676010132, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7124847173690796, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.45068737864494324, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4072861969470978, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.39693528413772583, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6334429383277893, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7678978443145752, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35670217871665955, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.41828110814094543, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.39440956711769104, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4834548532962799, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6731706261634827, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44254547357559204, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.36100220680236816, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3474372923374176, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6797668933868408, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6935744285583496, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4378896653652191, "metrics/hit_grouping/tracker/all/object_completeness": 0.3804553151130676, "metrics/hit_grouping/tracker/all/object_purity": 0.3670382797718048, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.648592472076416, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008262748830020428, "metrics/hit_grouping/tracker/background_recall": 0.03530178219079971, "metrics/hit_grouping/tracker/completeness": 0.39917805790901184, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5557198524475098, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6977116465568542, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4456492066383362, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.43984460830688477, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.43087178468704224, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6502050161361694, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6882891058921814, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45117077231407166, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3628150224685669, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3455491364002228, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6592423319816589, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6900060772895813, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.40655338764190674, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2673620879650116, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.24816876649856567, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6336973905563354, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7241696119308472, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45812365412712097, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.39867159724235535, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.371570885181427, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6081955432891846, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7531100511550903, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47203728556632996, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.414443701505661, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.37231799960136414, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5472702980041504, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6432483196258545, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4055256247520447, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35074782371520996, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.36020511388778687, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7190353274345398, "metrics/hit_grouping/tracker/purity": 0.38177689909935, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8931710720062256, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.007384882774204016, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.017184192314743996, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.018654117360711098, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0725456103682518, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6696011424064636, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.16485226154327393, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.16135238111019135, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.15406715869903564, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5925350189208984, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.669182300567627, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5274777412414551, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4547661542892456, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.43862801790237427, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7332506775856018, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.38853520154953003, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4149232506752014, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.4418179988861084, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5584986805915833, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.09719216823577881, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.027693942189216614, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.917144775390625, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26366478204727173, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1332249492406845, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7899287939071655, "metrics/particle/class_1/pid_accuracy": 0.9455451965332031, "metrics/particle/class_2/efficiency": 0.3594346046447754, "metrics/particle/class_2/pid_accuracy": 0.4300873875617981, "metrics/particle/class_3/efficiency": 0.6431682109832764, "metrics/particle/class_3/pid_accuracy": 0.9452301263809204, "metrics/particle/class_4/efficiency": 0.6371511220932007, "metrics/particle/class_4/pid_accuracy": 0.27341389656066895, "metrics/particle/class_5/efficiency": 0.7919999957084656, "metrics/particle/class_5/pid_accuracy": 0.3055555522441864, "metrics/particle/count/bias_mean": -2.9524738788604736, "metrics/particle/count/mae": 5.244140625, "metrics/particle/count/prediction_mean": 67.103515625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09623461216688156, "metrics/particle/event/energy_response_mean": 0.9640398025512695, "metrics/particle/event/met_abs_error": 9.490551948547363, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07957401126623154, "metrics/particle/event/scalar_pt_response_mean": 0.9720079898834229, "metrics/particle/event/vector_pt_closure": 0.07393833249807358, "metrics/particle/matched/delta_eta_abs_mean": 0.017774369567632675, "metrics/particle/matched/delta_phi_abs_mean": 0.016906581819057465, "metrics/particle/matched/delta_r_mean": 0.02745293267071247, "metrics/particle/matched/energy_relative_abs_mean": 0.18056604266166687, "metrics/particle/matched/pid_accuracy": 0.9137328863143921, "metrics/particle/matched/pt_relative_abs_mean": 0.17769968509674072, "metrics/particle/matching/duplicate_fraction": 0.08403915911912918, "metrics/particle/matching/efficiency": 0.6892273426055908, "metrics/particle/matching/f1": 0.7040635943412781, "metrics/particle/matching/purity": 0.7195525169372559, "metrics/particle/matching/target_coverage_dr0p05": 0.6281155347824097, "metrics/particle/matching/target_coverage_dr0p10": 0.7641023993492126, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.44923144578933716, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7436202168464661, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02610492706298828, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17906354367733002, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1768781989812851, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.012722072191536427, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21711966395378113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012125340290367603, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9793264865875244, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.017296653240919113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.5477254390716553, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7340.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.40540724992752075, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5767656564712524, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.032820701599121, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.815731406211853, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.21036918461322784, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.746389389038086, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025276146829128265, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5053445100784302, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04536699131131172, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8780009746551514, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10049.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6711777448654175, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.1987240314483643, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.303197860717773, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.0522565841674805, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.07136878371238708, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.15309467911720276, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.673515319824219, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2464863657951355, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.463321685791016, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02573576010763645, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18548068404197693, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18140709400177002, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.002734271576628089, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2233239859342575, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.035021889954805374, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.8289318084716797, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.036568980664014816, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.360129356384277, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1599.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6380763649940491, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.158308744430542, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.8475356101989746, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6369433403015137, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2551282048225403, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 16.459815979003906, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06923077255487442, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.129568099975586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09984157979488373, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.82878589630127, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2340.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5292949080467224, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.930316925048828, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.194808959960938, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7332350015640259, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06381852179765701, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08337526023387909, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.993505477905273, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.272188663482666, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.088573455810547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02798345871269703, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1773122102022171, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17449809610843658, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.00874065887182951, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2147534191608429, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009699948132038116, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3240000605583191, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0007467220420949161, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9110931158065796, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15464.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3058549463748932, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.741665005683899, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3771960735321045, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1598948240280151, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.20080456137657166, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.794607162475586, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007375125773251057, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6250532269477844, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0070113977417349815, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.310885429382324, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20881.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7213696241378784, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.3026444911956787, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.486140251159668, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.754045486450195, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.9437199831008911, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.1335509568452835, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.049842834472656, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7124897241592407, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.996063709259033, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027271147817373276, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17837420105934143, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1756667196750641, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.009544632397592068, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2160380333662033, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0065565709955990314, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6197211742401123, "metrics/particle/momentum/all/accepted/pz_bias": 0.0023331681732088327, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.929779052734375, "metrics/particle/momentum/all/accepted/samples": 24403.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3592904508113861, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6508285999298096, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6786378622055054, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4618724584579468, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.20751427114009857, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.805130958557129, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017132552340626717, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.067091464996338, "metrics/particle/momentum/all/assigned/pz_bias": 0.002280091866850853, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.8325726985931396, "metrics/particle/momentum/all/assigned/samples": 33270.0, "metrics/particle/momentum/all/efficiency": 0.6894671320915222, "metrics/particle/momentum/all/event/energy_bias": -7.8263959884643555, "metrics/particle/momentum/all/event/p3_error_mean": 26.318668365478516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.567475318908691, "metrics/particle/momentum/all/event/px_bias": 0.9512702822685242, "metrics/particle/momentum/all/event/py_bias": -0.06383145600557327, "metrics/particle/momentum/all/event/pz_abs_mean": 19.93294906616211, "metrics/particle/momentum/all/event/pz_bias": -2.3132123947143555, "metrics/particle/momentum/all/event/pz_residual_rms": 32.56081771850586, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029931414872407913, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17949460446834564, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17848607897758484, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.023465393111109734, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21755780279636383, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006882468704134226, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6943025588989258, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.02147796005010605, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.016263246536255, "metrics/particle/momentum/class_1/accepted/samples": 13222.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1971108317375183, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5355790257453918, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6287842392921448, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4056868553161621, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.13827022910118103, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.398578643798828, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.012682504020631313, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9301497340202332, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.060882046818733215, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7340915203094482, "metrics/particle/momentum/class_1/assigned/samples": 16164.0, "metrics/particle/momentum/class_1/efficiency": 0.7923533320426941, "metrics/particle/momentum/class_1/event/energy_bias": 7.920957565307617, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.470979690551758, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.32430648803711, "metrics/particle/momentum/class_1/event/px_bias": 0.6347846388816833, "metrics/particle/momentum/class_1/event/py_bias": 0.22171550989151, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.904131889343262, "metrics/particle/momentum/class_1/event/pz_bias": 0.8286805748939514, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.768112182617188, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038530316203832626, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20682752132415771, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20682716369628906, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04640156775712967, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24805743992328644, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0360531322658062, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8863016366958618, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.052078671753406525, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.370661735534668, "metrics/particle/momentum/class_2/accepted/samples": 1054.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8786382079124451, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9101361632347107, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0854926109313965, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6411328911781311, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37223973870277405, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.711758852005005, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06624605506658554, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.987804889678955, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06710625439882278, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.115779876708984, "metrics/particle/momentum/class_2/assigned/samples": 2219.0, "metrics/particle/momentum/class_2/efficiency": 0.3540476858615875, "metrics/particle/momentum/class_2/event/energy_bias": -19.365392684936523, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.332834243774414, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.324739456176758, "metrics/particle/momentum/class_2/event/px_bias": 0.1299656480550766, "metrics/particle/momentum/class_2/event/py_bias": -1.6606515645980835, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.53647232055664, "metrics/particle/momentum/class_2/event/pz_bias": -3.0647647380828857, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.482215881347656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02244800142943859, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1732456535100937, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1680641770362854, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014166921377182007, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2096574306488037, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0023543862625956535, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.351549357175827, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021506210789084435, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1638386249542236, "metrics/particle/momentum/class_3/accepted/samples": 9769.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4616718590259552, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.9755773544311523, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.9122071266174316, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.738227128982544, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2588987648487091, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.31646156311035, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013348164968192577, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8879762291908264, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05944519862532616, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.104790210723877, "metrics/particle/momentum/class_3/assigned/samples": 14384.0, "metrics/particle/momentum/class_3/efficiency": 0.6424015164375305, "metrics/particle/momentum/class_3/event/energy_bias": 8.131539344787598, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.561225891113281, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.17642879486084, "metrics/particle/momentum/class_3/event/px_bias": 0.326926052570343, "metrics/particle/momentum/class_3/event/py_bias": 0.969967246055603, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.398104667663574, "metrics/particle/momentum/class_3/event/pz_bias": -0.10906104743480682, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.754905700683594, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029655350372195244, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19261662662029266, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18343904614448547, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.01672343909740448, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22777967154979706, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.021551724523305893, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.145609974861145, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3056471347808838, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.9401159286499023, "metrics/particle/momentum/class_4/accepted/samples": 232.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45675230026245117, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.997804880142212, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.830855369567871, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.6456925868988037, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.266475647687912, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.141719818115234, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03151862323284149, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6531769037246704, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5835759043693542, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.852775573730469, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.6373626589775085, "metrics/particle/momentum/class_4/event/energy_bias": -2.5024592876434326, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.089860677719116, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.248361349105835, "metrics/particle/momentum/class_4/event/px_bias": 0.06983642280101776, "metrics/particle/momentum/class_4/event/py_bias": 0.26101234555244446, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.740724802017212, "metrics/particle/momentum/class_4/event/pz_bias": 0.23259814083576202, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.0923261642456055, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.023485086858272552, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19419337809085846, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19427862763404846, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06561724841594696, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23283205926418304, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02380952425301075, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.021886110305786, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.26166900992393494, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.731340169906616, "metrics/particle/momentum/class_5/accepted/samples": 126.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11494464427232742, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.32752394676208496, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.32303979992866516, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.017581455409526825, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.16883116960525513, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6191564798355103, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09740259498357773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.1665213108062744, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7008982300758362, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.654219150543213, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.7924528121948242, "metrics/particle/momentum/class_5/event/energy_bias": -2.011040449142456, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0441982746124268, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2275054454803467, "metrics/particle/momentum/class_5/event/px_bias": -0.2102426141500473, "metrics/particle/momentum/class_5/event/py_bias": 0.14412489533424377, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6635398864746094, "metrics/particle/momentum/class_5/event/pz_bias": -0.20066529512405396, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.2760844230651855, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4871457815170288, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.48901692032814026, "metrics/set_slot/angular_recall_all_dr0p10": 0.9226344227790833, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7595394253730774, "metrics/set_slot/soft_count_bias_mean": 1.1012828350067139}, "step_train_time": 5.654529094696045, "step_valid_time": 68766.04589056969, "step_total_time": 68771.7004199028, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006144062066923, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20466300352100253, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2948, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2760, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2707, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9117938553022795, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362279511533242, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9238493723849373, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06377204884667571, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8942847703997358, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9182496607869742, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9061087866108787, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9117938553022795, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9710738309766573, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19015134850604642, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2224, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2064, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2050, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9124668435013262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9280575539568345, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.920196165849309, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07194244604316546, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9062776304155614, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9217625899280576, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9139545251894784, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9124668435013262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.977988735122598, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19668840948451471, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1249, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1141, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1131, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8858695652173914, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9135308246597278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8994875837603469, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08646917534027222, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8781055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9055244195356285, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8916042569964525, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8858695652173914} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_28000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..e743cbaeb4b541a15238ef480e1325d070e9116b --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3265112753458403, "train": {"Classification_binary": 0.19733470678329468, "Cardinality": 0.13384483754634857, "Classification": 0.11025698482990265, "Regression_pt": 0.23774893581867218, "Regression_eta": 0.0021662265062332153, "Regression_sin_phi": 0.0012162893544882536, "Regression_cos_phi": 0.0011012546019628644, "Regression_energy": 0.23747657239437103, "HitGrouping": 0.2454470694065094, "Auxiliary": 0.3026253581047058, "Total": 1.429717779159546, "Total_unweighted": 0.7873009702889249}, "train_group": {}, "valid": {"Classification_binary": 0.226357102394104, "Cardinality": 0.15587331851323447, "Classification": 0.10868896047274272, "Regression_pt": 0.2410197655359904, "Regression_eta": 0.0022049257531762123, "Regression_sin_phi": 0.001226857692624132, "Regression_cos_phi": 0.0012386638360718887, "Regression_energy": 0.2403913935025533, "HitGrouping": 0.2474064032236735, "Auxiliary": 0.3223916292190552, "Total": 1.5173060099283855, "Total_unweighted": 0.8211276691872627, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.724146842956543, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.31859150528907776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.44677433371543884, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.30770477652549744, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3050661087036133, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6243506073951721, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6763096451759338, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2804467976093292, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4103693664073944, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.267796128988266, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2968776822090149, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6946797966957092, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.744534432888031, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3099401295185089, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4174155592918396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.30113351345062256, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2904287576675415, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5894535779953003, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7318543195724487, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3100786507129669, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4262309968471527, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3002578914165497, "metrics/hit_grouping/calorimeter/all/object_purity": 0.29571738839149475, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6101216077804565, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02567049115896225, "metrics/hit_grouping/calorimeter/background_recall": 0.627799928188324, "metrics/hit_grouping/calorimeter/completeness": 0.3545900285243988, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3786119222640991, "metrics/hit_grouping/calorimeter/energy_purity": 0.37418511509895325, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5301921963691711, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7579934000968933, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3282756209373474, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3609435558319092, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3211115002632141, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.32086363434791565, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5528217554092407, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.71761554479599, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.32167449593544006, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.47925764322280884, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3095341622829437, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3039771020412445, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6455603837966919, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.6913645267486572, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2594777047634125, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5091644525527954, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.24645081162452698, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.23388372361660004, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6944760680198669, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7058948874473572, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3190528452396393, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5083774328231812, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.30810222029685974, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.30692732334136963, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6657739281654358, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7140197157859802, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.40971460938453674, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6263736486434937, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.39779385924339294, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.39150646328926086, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6305599212646484, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7620445489883423, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.28768110275268555, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3123074769973755, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.27934369444847107, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2714085578918457, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5476764440536499, "metrics/hit_grouping/calorimeter/purity": 0.34932488203048706, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8472185730934143, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06653158366680145, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14923076331615448, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06379487365484238, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.023113273084163666, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05384615436196327, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7744463682174683, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12568537890911102, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11117445677518845, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.12291311472654343, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.11949020624160767, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4104903042316437, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7269457578659058, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3258751928806305, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.45025578141212463, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3154980540275574, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.31166359782218933, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6334601044654846, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7100721597671509, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4441123604774475, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.39818650484085083, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.38768506050109863, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6331440806388855, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7601826786994934, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35221537947654724, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4179910123348236, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.40622925758361816, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48401570320129395, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6620887517929077, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4416791498661041, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3455570638179779, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3407784700393677, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6871702075004578, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6855161786079407, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4349140226840973, "metrics/hit_grouping/tracker/all/object_completeness": 0.36829206347465515, "metrics/hit_grouping/tracker/all/object_purity": 0.36111652851104736, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6529850959777832, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014945540577173233, "metrics/hit_grouping/tracker/background_recall": 0.05508499592542648, "metrics/hit_grouping/tracker/completeness": 0.3868490755558014, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5509435534477234, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6878783702850342, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4355353116989136, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4285958409309387, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4309729039669037, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6569476127624512, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.676692545413971, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45079001784324646, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.34499332308769226, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3356391489505768, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6668570637702942, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6896477341651917, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4166666567325592, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2591809034347534, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.23311468958854675, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6294498443603516, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7147488594055176, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4504852592945099, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3799354135990143, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3623162508010864, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6185298562049866, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7526240944862366, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.49334222078323364, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4250694215297699, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.36368435621261597, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5392809510231018, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.635633647441864, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4043549597263336, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.34314027428627014, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3591021001338959, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7178646922111511, "metrics/hit_grouping/tracker/purity": 0.3724954128265381, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8811782598495483, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012163336388766766, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.022127404808998108, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.023914314806461334, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6699420809745789, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1736651062965393, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.15675336122512817, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.15977026522159576, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5821669101715088, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6607024073600769, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5221061110496521, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.43941712379455566, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4298631548881531, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7393896579742432, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4270673990249634, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5053226351737976, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5552846193313599, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5967261791229248, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05865991860628128, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06592141836881638, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.219985961914062, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.24406498670578003, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.11362515389919281, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7591610550880432, "metrics/particle/class_1/pid_accuracy": 0.9403930306434631, "metrics/particle/class_2/efficiency": 0.3700920045375824, "metrics/particle/class_2/pid_accuracy": 0.47135496139526367, "metrics/particle/class_3/efficiency": 0.6602664589881897, "metrics/particle/class_3/pid_accuracy": 0.9508724212646484, "metrics/particle/class_4/efficiency": 0.6111645698547363, "metrics/particle/class_4/pid_accuracy": 0.26929134130477905, "metrics/particle/class_5/efficiency": 0.7300000190734863, "metrics/particle/class_5/pid_accuracy": 0.3287671208381653, "metrics/particle/count/bias_mean": -2.5162761211395264, "metrics/particle/count/mae": 4.925130367279053, "metrics/particle/count/prediction_mean": 67.53971099853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09252563118934631, "metrics/particle/event/energy_response_mean": 0.9604857563972473, "metrics/particle/event/met_abs_error": 10.244073867797852, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07356467097997665, "metrics/particle/event/scalar_pt_response_mean": 0.9828150868415833, "metrics/particle/event/vector_pt_closure": 0.07675840705633163, "metrics/particle/matched/delta_eta_abs_mean": 0.017618024721741676, "metrics/particle/matched/delta_phi_abs_mean": 0.01778908260166645, "metrics/particle/matched/delta_r_mean": 0.02801363356411457, "metrics/particle/matched/energy_relative_abs_mean": 0.18226541578769684, "metrics/particle/matched/pid_accuracy": 0.9148738980293274, "metrics/particle/matched/pt_relative_abs_mean": 0.18062621355056763, "metrics/particle/matching/duplicate_fraction": 0.08679307252168655, "metrics/particle/matching/efficiency": 0.6825270056724548, "metrics/particle/matching/f1": 0.6950086951255798, "metrics/particle/matching/purity": 0.7079553604125977, "metrics/particle/matching/target_coverage_dr0p05": 0.626015305519104, "metrics/particle/matching/target_coverage_dr0p10": 0.7676430940628052, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.43884170055389404, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7368176579475403, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026620181277394295, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1790035516023636, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17881274223327637, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.002628566464409232, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22050391137599945, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013513513840734959, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0282578468322754, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.037926025688648224, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.6729350090026855, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7252.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.39835307002067566, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4439665079116821, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7374674081802368, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5267008543014526, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.22468574345111847, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.264467239379883, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02533900737762451, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.518652319908142, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.013968863524496555, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.778074264526367, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10103.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6631309390068054, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.51557993888855, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.459641456604004, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.109858512878418, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.25881826877593994, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.164901852607727, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.852157592773438, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0483307838439941, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.554227828979492, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025517599657177925, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18652111291885376, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1828545331954956, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003506111679598689, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22408659756183624, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04297114908695221, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.8919733762741089, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16774842143058777, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.460853099822998, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1629.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6187717914581299, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5144047737121582, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.480661392211914, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.278281211853027, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27149322628974915, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 72.06327056884766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08144796639680862, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.298275947570801, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.027577856555581093, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.793667793273926, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2431.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5392253994941711, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.545825958251953, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.164252281188965, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.833540916442871, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04903257638216019, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10691754519939423, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.9470853805542, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.4314324855804443, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.949081420898438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028802167624235153, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18369139730930328, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1814151555299759, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0058522396720945835, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.22210295498371124, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009799437830224633, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3266373574733734, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010012013837695122, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9017114043235779, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15307.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3141880929470062, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7776572704315186, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.494866132736206, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.2848551273345947, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2133416384458542, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.318758010864258, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0073268841952085495, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6326567530632019, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.03727707639336586, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.2501018047332764, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20882.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7140458226203918, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5481829643249512, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.726217269897461, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.973794937133789, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.9662517309188843, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.2045583724975586, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.390803337097168, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.445522665977478, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.743552207946777, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027926761656999588, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1824764609336853, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1807318478822708, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0031515341252088547, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22175918519496918, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007565734907984734, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6424172520637512, "metrics/particle/momentum/all/accepted/pz_bias": 0.006409428082406521, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9991955757141113, "metrics/particle/momentum/all/accepted/samples": 24188.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3617928922176361, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6576176881790161, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7854297161102295, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.5757453441619873, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.22100190818309784, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.789810180664062, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018164951354265213, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0944514274597168, "metrics/particle/momentum/all/assigned/pz_bias": -0.021077753975987434, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.8022751808166504, "metrics/particle/momentum/all/assigned/samples": 33416.0, "metrics/particle/momentum/all/efficiency": 0.6833926439285278, "metrics/particle/momentum/all/event/energy_bias": -10.513222694396973, "metrics/particle/momentum/all/event/p3_error_mean": 26.385618209838867, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.595242500305176, "metrics/particle/momentum/all/event/px_bias": 1.1760374307632446, "metrics/particle/momentum/all/event/py_bias": 4.476377487182617, "metrics/particle/momentum/all/event/pz_abs_mean": 19.044565200805664, "metrics/particle/momentum/all/event/pz_bias": -4.925285816192627, "metrics/particle/momentum/all/event/pz_residual_rms": 30.139001846313477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03134160861372948, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1878458857536316, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1861337572336197, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0003184836823493242, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2264660745859146, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.008587410673499107, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7370777726173401, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.016418615356087685, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.113243818283081, "metrics/particle/momentum/class_1/accepted/samples": 12693.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2160450518131256, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6335607171058655, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7023221850395203, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.49061277508735657, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1732979118824005, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.972556114196777, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014704065397381783, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0320398807525635, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.00019839253218378872, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.6879353523254395, "metrics/particle/momentum/class_1/assigned/samples": 16186.0, "metrics/particle/momentum/class_1/efficiency": 0.7606520056724548, "metrics/particle/momentum/class_1/event/energy_bias": 3.7502596378326416, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.791616439819336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.9684419631958, "metrics/particle/momentum/class_1/event/px_bias": 0.6590776443481445, "metrics/particle/momentum/class_1/event/py_bias": 1.1506645679473877, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.455920219421387, "metrics/particle/momentum/class_1/event/pz_bias": -1.6332790851593018, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.003793716430664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037188634276390076, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21441248059272766, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21537363529205322, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.050648972392082214, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25603488087654114, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.036245353519916534, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.018878936767578, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11952490359544754, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.5475969314575195, "metrics/particle/momentum/class_2/accepted/samples": 1076.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.873827338218689, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8791794776916504, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2347445487976074, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7775694131851196, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.394527792930603, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.111180305480957, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06663724780082703, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0231003761291504, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.19813334941864014, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.93671178817749, "metrics/particle/momentum/class_2/assigned/samples": 2266.0, "metrics/particle/momentum/class_2/efficiency": 0.36143767833709717, "metrics/particle/momentum/class_2/event/energy_bias": -19.87782096862793, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.73964500427246, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.685993194580078, "metrics/particle/momentum/class_2/event/px_bias": 1.5960488319396973, "metrics/particle/momentum/class_2/event/py_bias": -1.4008723497390747, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.642398834228516, "metrics/particle/momentum/class_2/event/pz_bias": 0.9973090887069702, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.646514892578125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02260471135377884, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17109668254852295, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1692933589220047, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014055689796805382, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2105632871389389, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002483855001628399, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.33961525559425354, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.006655123550444841, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1630228757858276, "metrics/particle/momentum/class_3/accepted/samples": 10065.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4449952244758606, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.8950605392456055, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.069255828857422, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.900851011276245, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24581778049468994, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 34.080631256103516, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013341628946363926, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8254972696304321, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.08112645894289017, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.131211996078491, "metrics/particle/momentum/class_3/assigned/samples": 14466.0, "metrics/particle/momentum/class_3/efficiency": 0.6618662476539612, "metrics/particle/momentum/class_3/event/energy_bias": 9.685249328613281, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.538259506225586, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.677803993225098, "metrics/particle/momentum/class_3/event/px_bias": -1.074067234992981, "metrics/particle/momentum/class_3/event/py_bias": 4.213362693786621, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.407763481140137, "metrics/particle/momentum/class_3/event/pz_bias": -4.450883388519287, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.13449478149414, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03138355910778046, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20778247714042664, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19662389159202576, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.010938141494989395, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23762863874435425, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.021739130839705467, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.33420991897583, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.301590234041214, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.9756975173950195, "metrics/particle/momentum/class_4/accepted/samples": 230.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.46458858251571655, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.4539003372192383, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.973719358444214, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.817294120788574, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3025936484336853, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.021484375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02593659982085228, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.615994930267334, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3783484101295471, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.179647445678711, "metrics/particle/momentum/class_4/assigned/samples": 347.0, "metrics/particle/momentum/class_4/efficiency": 0.6318681240081787, "metrics/particle/momentum/class_4/event/energy_bias": -2.5654654502868652, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.895331382751465, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7887680530548096, "metrics/particle/momentum/class_4/event/px_bias": 0.311481237411499, "metrics/particle/momentum/class_4/event/py_bias": 0.20072343945503235, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.2524943351745605, "metrics/particle/momentum/class_4/event/pz_bias": 0.2816813886165619, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.068002223968506, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02357988804578781, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23247607052326202, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22615234553813934, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07432391494512558, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2654571533203125, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04032257944345474, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.303624153137207, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.4928305745124817, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.102202415466309, "metrics/particle/momentum/class_5/accepted/samples": 124.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09377354383468628, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4335506558418274, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.42689383029937744, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07129848748445511, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.16556291282176971, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.3569022417068481, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10596026480197906, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.409623622894287, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7564703226089478, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.320565223693848, "metrics/particle/momentum/class_5/assigned/samples": 151.0, "metrics/particle/momentum/class_5/efficiency": 0.7798742055892944, "metrics/particle/momentum/class_5/event/energy_bias": -1.5054447650909424, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.4811296463012695, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.612452983856201, "metrics/particle/momentum/class_5/event/px_bias": -0.3165031373500824, "metrics/particle/momentum/class_5/event/py_bias": 0.31249940395355225, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.8794482946395874, "metrics/particle/momentum/class_5/event/pz_bias": -0.12011383473873138, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.786374568939209, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4934888184070587, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.49586430191993713, "metrics/set_slot/angular_recall_all_dr0p10": 0.9276527166366577, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7584056854248047, "metrics/set_slot/soft_count_bias_mean": 1.1234239339828491}, "step_train_time": 4.960845470428467, "step_valid_time": 71315.89569807053, "step_total_time": 71320.85654377937, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988927325424439, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21274766412860102, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3004, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2789, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2724, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9213742979848034, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9284287616511319, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9248880782623113, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07157123834886818, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8999008919722498, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9067909454061251, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9033327806333942, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9213742979848034, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9832843291686838, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1919932008643881, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2079, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2061, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9190981432360743, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9318691169878978, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9254395726685956, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0681308830121022, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9111405835543767, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9238009861048857, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.917427108835967, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9190981432360743, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9923058450703488, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18375133190500592, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1264, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1156, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1144, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8975155279503105, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9145569620253164, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9059561128526645, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08544303797468354, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8881987577639752, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9050632911392406, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.896551724137931, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8975155279503105} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_29000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..9d0877485161417e116588d8cc3406bd2745b51e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3738866780367633, "train": {"Classification_binary": 0.18898245692253113, "Cardinality": 0.14041876792907715, "Classification": 0.099112369120121, "Regression_pt": 0.21289965510368347, "Regression_eta": 0.0019369126530364156, "Regression_sin_phi": 0.0009828677866607904, "Regression_cos_phi": 0.0010220954427495599, "Regression_energy": 0.2133505642414093, "HitGrouping": 0.24269014596939087, "Auxiliary": 0.29264384508132935, "Total": 1.3527770042419434, "Total_unweighted": 0.7182869212701917}, "train_group": {}, "valid": {"Classification_binary": 0.2182526191075643, "Cardinality": 0.15511522690455118, "Classification": 0.10341825087865193, "Regression_pt": 0.21818049748738608, "Regression_eta": 0.0020871752252181372, "Regression_sin_phi": 0.0011448035947978497, "Regression_cos_phi": 0.0011387626485278208, "Regression_energy": 0.2179363171259562, "HitGrouping": 0.24492287635803223, "Auxiliary": 0.3062419692675273, "Total": 1.446441650390625, "Total_unweighted": 0.7621584260681024, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7154518365859985, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3179042339324951, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4505525529384613, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.30633604526519775, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3088635206222534, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6429583430290222, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6521254181861877, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2669425308704376, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4174855947494507, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.25459232926368713, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.29746773838996887, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7299220561981201, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7436283230781555, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3126828372478485, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4244396388530731, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.304246187210083, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.29533135890960693, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5971490740776062, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7263538837432861, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3102603554725647, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.43221884965896606, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.300464928150177, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2998775541782379, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6237993836402893, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025445908308029175, "metrics/hit_grouping/calorimeter/background_recall": 0.6264181137084961, "metrics/hit_grouping/calorimeter/completeness": 0.3573373556137085, "metrics/hit_grouping/calorimeter/energy_completeness": 0.38465985655784607, "metrics/hit_grouping/calorimeter/energy_purity": 0.3801024854183197, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5360713601112366, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7497413754463196, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.32693997025489807, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.36673635244369507, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3185269832611084, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.32223761081695557, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5716333389282227, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7157210111618042, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.31886813044548035, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.47586122155189514, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.30825671553611755, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.30934861302375793, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6490781307220459, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.6879307627677917, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.26598483324050903, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5253444314002991, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.25411367416381836, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.24267880618572235, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.70787513256073, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7031607627868652, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3276367485523224, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5220803618431091, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3170103132724762, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3132765293121338, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6706944704055786, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7032959461212158, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4182555675506592, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6562009453773499, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.40111321210861206, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.40554091334342957, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6692830920219421, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7542386651039124, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.27776017785072327, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.30695056915283203, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.26979830861091614, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.27195173501968384, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5675639510154724, "metrics/hit_grouping/calorimeter/purity": 0.3519502878189087, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8601135015487671, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05744214728474617, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1584615409374237, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05939926579594612, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.022580640390515327, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.777582585811615, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.11853333562612534, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11972633749246597, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.11699803173542023, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.11459346115589142, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4110604226589203, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7205564379692078, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.32667630910873413, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4560270309448242, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3161553144454956, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3164446949958801, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6483145356178284, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7183161973953247, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4324566721916199, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4103211760520935, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4062924385070801, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6264196038246155, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7715197205543518, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3589456081390381, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.43459293246269226, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.42384007573127747, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48850253224372864, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.673023521900177, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44372686743736267, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3645302653312683, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3563906252384186, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.687957763671875, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6956344246864319, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4330247640609741, "metrics/hit_grouping/tracker/all/object_completeness": 0.38490405678749084, "metrics/hit_grouping/tracker/all/object_purity": 0.3778437674045563, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.651709794998169, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006820046342909336, "metrics/hit_grouping/tracker/background_recall": 0.032754842191934586, "metrics/hit_grouping/tracker/completeness": 0.40346139669418335, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5499330163002014, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7028497457504272, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42460137605667114, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.44108864665031433, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.44156667590141296, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6503872275352478, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6898623704910278, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45117077231407166, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.361290842294693, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.35275396704673767, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6620978713035583, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6857502460479736, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43244338035583496, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.28527122735977173, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.26304543018341064, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6436083912849426, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.721811830997467, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45839324593544006, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.40263181924819946, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.38638728857040405, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6127786040306091, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7423146963119507, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4866844117641449, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4301268458366394, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.39231953024864197, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5699067711830139, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6533238291740417, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39054086804389954, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35385772585868835, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3641683757305145, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7168110609054565, "metrics/hit_grouping/tracker/purity": 0.3853129744529724, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8916333913803101, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013466550968587399, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.025832848623394966, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.024550722911953926, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6799376606941223, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.17677552998065948, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.16915683448314667, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.16399987041950226, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5775012969970703, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6707887649536133, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5192137360572815, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4582621157169342, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4501998722553253, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.738327145576477, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.40112337470054626, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5269820690155029, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5798702836036682, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5716525912284851, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08460398018360138, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0408477745950222, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.999025344848633, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28825756907463074, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15781773626804352, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7905025482177734, "metrics/particle/class_1/pid_accuracy": 0.9479625821113586, "metrics/particle/class_2/efficiency": 0.3857976198196411, "metrics/particle/class_2/pid_accuracy": 0.5199185609817505, "metrics/particle/class_3/efficiency": 0.6498616337776184, "metrics/particle/class_3/pid_accuracy": 0.9452330470085144, "metrics/particle/class_4/efficiency": 0.6333012580871582, "metrics/particle/class_4/pid_accuracy": 0.29331305623054504, "metrics/particle/class_5/efficiency": 0.765999972820282, "metrics/particle/class_5/pid_accuracy": 0.33681461215019226, "metrics/particle/count/bias_mean": -3.12890625, "metrics/particle/count/mae": 5.18359375, "metrics/particle/count/prediction_mean": 66.92708587646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.0867752730846405, "metrics/particle/event/energy_response_mean": 0.9813984036445618, "metrics/particle/event/met_abs_error": 9.080109596252441, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07215295732021332, "metrics/particle/event/scalar_pt_response_mean": 0.991601288318634, "metrics/particle/event/vector_pt_closure": 0.06999033689498901, "metrics/particle/matched/delta_eta_abs_mean": 0.017230572178959846, "metrics/particle/matched/delta_phi_abs_mean": 0.016922611743211746, "metrics/particle/matched/delta_r_mean": 0.02700130082666874, "metrics/particle/matched/energy_relative_abs_mean": 0.18008793890476227, "metrics/particle/matched/pid_accuracy": 0.9182592034339905, "metrics/particle/matched/pt_relative_abs_mean": 0.17827191948890686, "metrics/particle/matching/duplicate_fraction": 0.08168287575244904, "metrics/particle/matching/efficiency": 0.6944222450256348, "metrics/particle/matching/f1": 0.7102839350700378, "metrics/particle/matching/purity": 0.7268871665000916, "metrics/particle/matching/target_coverage_dr0p05": 0.6469806432723999, "metrics/particle/matching/target_coverage_dr0p10": 0.7765644788742065, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4497053921222687, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.74407559633255, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02568878047168255, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17542774975299835, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17446789145469666, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014079445973038673, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21538963913917542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012061403132975101, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.984828531742096, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0011268488597124815, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.541186571121216, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7296.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3734057545661926, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.790645956993103, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.0827834606170654, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.8980051279067993, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2151784747838974, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 21.734195709228516, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025197479873895645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5090973377227783, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.007462412118911743, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9802651405334473, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10001.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6671543717384338, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.975662112236023, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.680512428283691, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.927584648132324, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.02940378338098526, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4173792898654938, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.990203857421875, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6111235618591309, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.126855850219727, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026134341955184937, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1798798143863678, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1803910732269287, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02757468819618225, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22039690613746643, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.040098704397678375, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.9497374296188354, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.24016304314136505, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.793704509735107, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1621.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6024101972579956, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.747092604637146, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.1429336071014404, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.9639925956726074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2630917429924011, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.937591552734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07750314474105835, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.276906728744507, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.21252737939357758, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.475618362426758, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2387.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5365772843360901, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.8729751110076904, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.433704376220703, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6336370706558228, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07294294238090515, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.3052263855934143, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.249334335327148, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.9734666347503662, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.158483505249023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02757752686738968, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18120941519737244, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17848509550094604, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0026530977338552475, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2191813886165619, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0012161557096987963, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3224249482154846, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03181374445557594, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9342628121376038, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15623.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2884620726108551, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6792752742767334, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3662852048873901, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.161446213722229, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.19966410100460052, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 15.943971633911133, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0070057581178843975, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6066587567329407, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01975594460964203, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.209019422531128, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20840.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7287867069244385, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.11360135674476624, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.036303520202637, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.044413566589355, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.07260997593402863, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8779075145721436, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.447115898132324, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6047831773757935, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.040995597839355, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026920655742287636, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1794026494026184, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1774166375398636, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007696476764976978, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21814186871051788, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007008965127170086, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6268573999404907, "metrics/particle/momentum/all/accepted/pz_bias": -0.03578276187181473, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.998382568359375, "metrics/particle/momentum/all/accepted/samples": 24540.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.33658161759376526, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7176674604415894, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7095668315887451, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.512626051902771, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.20889009535312653, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.093509674072266, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017545443028211594, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0700982809066772, "metrics/particle/momentum/all/assigned/pz_bias": -0.029903946444392204, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6040658950805664, "metrics/particle/momentum/all/assigned/samples": 33228.0, "metrics/particle/momentum/all/efficiency": 0.6933378577232361, "metrics/particle/momentum/all/event/energy_bias": -4.735034942626953, "metrics/particle/momentum/all/event/p3_error_mean": 24.523775100708008, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.520126342773438, "metrics/particle/momentum/all/event/px_bias": -0.11614909768104553, "metrics/particle/momentum/all/event/py_bias": -2.600513219833374, "metrics/particle/momentum/all/event/pz_abs_mean": 17.74989891052246, "metrics/particle/momentum/all/event/pz_bias": -3.189373731613159, "metrics/particle/momentum/all/event/pz_residual_rms": 26.98013687133789, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02901693992316723, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1830773502588272, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18136084079742432, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002401212928816676, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2215847671031952, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007179564796388149, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7032110691070557, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04156270995736122, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.0567400455474854, "metrics/particle/momentum/class_1/accepted/samples": 13232.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.17521020770072937, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.49597686529159546, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5402739644050598, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.34683412313461304, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.14759258925914764, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.233640670776367, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011543209664523602, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9043731093406677, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.020346755161881447, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.584012031555176, "metrics/particle/momentum/class_1/assigned/samples": 16200.0, "metrics/particle/momentum/class_1/efficiency": 0.7929525971412659, "metrics/particle/momentum/class_1/event/energy_bias": 7.470629692077637, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.33529281616211, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.382173538208008, "metrics/particle/momentum/class_1/event/px_bias": 0.8003334403038025, "metrics/particle/momentum/class_1/event/py_bias": -2.4506702423095703, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.530508995056152, "metrics/particle/momentum/class_1/event/pz_bias": -1.5188769102096558, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.704477310180664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03709534555673599, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20641255378723145, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2041645497083664, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.034059278666973114, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24415795505046844, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03870387002825737, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9398455619812012, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.042017482221126556, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.522922515869141, "metrics/particle/momentum/class_2/accepted/samples": 1111.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7616678476333618, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7921929359436035, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0223839282989502, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6126601696014404, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36384129524230957, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.540696382522583, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.064472496509552, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9412293434143066, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07736100256443024, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.012982368469238, "metrics/particle/momentum/class_2/assigned/samples": 2218.0, "metrics/particle/momentum/class_2/efficiency": 0.37319448590278625, "metrics/particle/momentum/class_2/event/energy_bias": -11.772059440612793, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.319053649902344, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.554306983947754, "metrics/particle/momentum/class_2/event/px_bias": -0.2699163258075714, "metrics/particle/momentum/class_2/event/py_bias": -0.7889934778213501, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.29194450378418, "metrics/particle/momentum/class_2/event/pz_bias": -0.5509787797927856, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.588871002197266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022897951304912567, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17074675858020782, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16867069900035858, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020692486315965652, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20976296067237854, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002438776660710573, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3409457206726074, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.023373618721961975, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2426578998565674, "metrics/particle/momentum/class_3/accepted/samples": 9841.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.45408880710601807, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.218024969100952, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.118330478668213, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.951547145843506, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.25248077511787415, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.224872589111328, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01593291386961937, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9346561431884766, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.029880225658416748, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.771270513534546, "metrics/particle/momentum/class_3/assigned/samples": 14310.0, "metrics/particle/momentum/class_3/efficiency": 0.6471362113952637, "metrics/particle/momentum/class_3/event/energy_bias": 4.4109907150268555, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.913497924804688, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.46005916595459, "metrics/particle/momentum/class_3/event/px_bias": -1.3187479972839355, "metrics/particle/momentum/class_3/event/py_bias": 0.5329082012176514, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.102327346801758, "metrics/particle/momentum/class_3/event/pz_bias": -1.6980063915252686, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.793245315551758, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.031175069510936737, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18508175015449524, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17553910613059998, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0005586420884355903, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21524731814861298, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.012500000186264515, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.185759425163269, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04199237748980522, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6933369636535645, "metrics/particle/momentum/class_4/accepted/samples": 240.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4040953814983368, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.373443603515625, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.173574209213257, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.0121910572052, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.28205129504203796, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.550606727600098, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017094017937779427, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3686848878860474, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0461636520922184, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6089932918548584, "metrics/particle/momentum/class_4/assigned/samples": 351.0, "metrics/particle/momentum/class_4/efficiency": 0.6593406796455383, "metrics/particle/momentum/class_4/event/energy_bias": -2.592491626739502, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.591200828552246, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6022675037384033, "metrics/particle/momentum/class_4/event/px_bias": 0.36132729053497314, "metrics/particle/momentum/class_4/event/py_bias": -0.028837110847234726, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0363383293151855, "metrics/particle/momentum/class_4/event/pz_bias": 0.5100399851799011, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.875134468078613, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.022819140926003456, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22412408888339996, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21718139946460724, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.073811836540699, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2578086853027344, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06034482643008232, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.441309928894043, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.5304151177406311, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.512993812561035, "metrics/particle/momentum/class_5/accepted/samples": 116.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10940562188625336, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3270428478717804, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.32323548197746277, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0669364482164383, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.20805369317531586, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4113832414150238, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.12751677632331848, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.5395736694335938, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.544037401676178, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.480651378631592, "metrics/particle/momentum/class_5/assigned/samples": 149.0, "metrics/particle/momentum/class_5/efficiency": 0.7295597195625305, "metrics/particle/momentum/class_5/event/energy_bias": -2.252105712890625, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.2249279022216797, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.465836524963379, "metrics/particle/momentum/class_5/event/px_bias": 0.310854434967041, "metrics/particle/momentum/class_5/event/py_bias": 0.13507936894893646, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6891374588012695, "metrics/particle/momentum/class_5/event/pz_bias": 0.06844889372587204, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.095161437988281, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.48148104548454285, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.482319712638855, "metrics/set_slot/angular_recall_all_dr0p10": 0.9303291440010071, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7734048366546631, "metrics/set_slot/soft_count_bias_mean": 0.8351691365242004}, "step_train_time": 4.6305084228515625, "step_valid_time": 73867.63996744156, "step_total_time": 73872.27047634125, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007537136640332, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18937318564094485, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2994, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2793, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2751, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9226957383548068, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9328657314629258, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9277528649725959, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06713426853707415, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.908820614469772, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9188376753507014, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9138016940707524, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9226957383548068, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9957647645550414, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18218445343298972, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2075, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2060, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9173297966401415, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9267530147387226, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9220173294823373, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07324698526127736, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9106984969053935, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9200535953550693, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9153521439680071, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9173297966401415, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9874948021393599, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1855247968863707, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1272, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1165, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9045031055900621, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9158805031446541, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.91015625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08411949685534591, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8920807453416149, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9033018867924528, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.89765625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9045031055900621} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_3000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..b0fa2c095792efadcf1149fc17948eedf3fdc86d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14212620807276863, "train": {"Classification_binary": 0.381303608417511, "Cardinality": 0.2886629104614258, "Classification": 0.2940569519996643, "Regression_pt": 0.4663598835468292, "Regression_eta": 0.005993862170726061, "Regression_sin_phi": 0.002729986794292927, "Regression_cos_phi": 0.0027653733268380165, "Regression_energy": 0.46349915862083435, "HitGrouping": 0.3071323037147522, "Auxiliary": 0.5327242016792297, "Total": 2.7668280601501465, "Total_unweighted": 1.6167088248766959}, "train_group": {}, "valid": {"Classification_binary": 0.38801129659016925, "Cardinality": 0.24406679471333823, "Classification": 0.3129770557085673, "Regression_pt": 0.4769500096638997, "Regression_eta": 0.00627611701687177, "Regression_sin_phi": 0.0028919040535887084, "Regression_cos_phi": 0.0028775244330366454, "Regression_energy": 0.4705563386281331, "HitGrouping": 0.3110549847284953, "Auxiliary": 0.5259150266647339, "Total": 2.7765671412150064, "Total_unweighted": 1.6605402460942664, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8126451969146729, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.08002264052629471, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.19930103421211243, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.07778367400169373, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.05209692195057869, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4747331738471985, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8087483048439026, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.0852903351187706, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.17485597729682922, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.08074380457401276, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.04892142117023468, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4384954273700714, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7806646823883057, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.08176320046186447, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.20065076649188995, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.07953424006700516, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.06209801509976387, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5584133863449097, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7934747338294983, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.0815194696187973, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.19790273904800415, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.07907941192388535, "metrics/hit_grouping/calorimeter/all/object_purity": 0.0576978400349617, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5207294821739197, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026934541761875153, "metrics/hit_grouping/calorimeter/background_recall": 0.6280590295791626, "metrics/hit_grouping/calorimeter/completeness": 0.1887357234954834, "metrics/hit_grouping/calorimeter/energy_completeness": 0.23580726981163025, "metrics/hit_grouping/calorimeter/energy_purity": 0.2333555966615677, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5989221334457397, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7997486591339111, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.07145125418901443, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.15258285403251648, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.06886150687932968, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.06443925201892853, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4959092140197754, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7956852316856384, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.0806674063205719, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.20172245800495148, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.07831287384033203, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.05272339656949043, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5192867517471313, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7778915762901306, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.10371851176023483, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.28984957933425903, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.10150618106126785, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.0486123152077198, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5747693181037903, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.774506151676178, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.08920738846063614, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.2480224221944809, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.08575238287448883, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.062476735562086105, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5701027512550354, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8083493113517761, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3441454768180847, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5970696210861206, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.34400203824043274, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.20268113911151886, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5049712061882019, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8119637966156006, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.03964803367853165, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.09294228255748749, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.03800523653626442, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.034007713198661804, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4696665406227112, "metrics/hit_grouping/calorimeter/purity": 0.18617480993270874, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8882436156272888, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.027691658586263657, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10153846442699432, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.026681318879127502, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0027076080441474915, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.828087329864502, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.027654211968183517, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.052451539784669876, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.026603223755955696, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.017950061708688736, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3437856435775757, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7894641160964966, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.08576486259698868, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2083224058151245, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.08321444690227509, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.06115603819489479, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5411529541015625, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6668440103530884, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.37567245960235596, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.21420858800411224, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.18383602797985077, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7136879563331604, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7668057680130005, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.36006730794906616, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.28403303027153015, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.19389715790748596, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5047672390937805, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.611172616481781, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.3763881325721741, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.19291885197162628, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.1638946831226349, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.760100781917572, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6418758034706116, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.37478744983673096, "metrics/hit_grouping/tracker/all/object_completeness": 0.20732125639915466, "metrics/hit_grouping/tracker/all/object_purity": 0.17272436618804932, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7239278554916382, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0035015265457332134, "metrics/hit_grouping/tracker/background_recall": 0.009358526207506657, "metrics/hit_grouping/tracker/completeness": 0.21870960295200348, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6400347948074341, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6336724162101746, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3938041031360626, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.25412508845329285, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.22573386132717133, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7415945529937744, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.640719473361969, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.3542737364768982, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.17328430712223053, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.14002090692520142, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.733676016330719, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6613147854804993, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3543689250946045, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1395874172449112, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.0897979587316513, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.674352765083313, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6726759672164917, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4033968448638916, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.21126753091812134, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.166853129863739, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6934759020805359, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6977944374084473, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.39280959963798523, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.22407867014408112, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.17275725305080414, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6358188986778259, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5919186472892761, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3343479335308075, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.19923371076583862, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.18036727607250214, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7790915369987488, "metrics/hit_grouping/tracker/purity": 0.20817607641220093, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8945377469062805, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0095569072291255, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.008898059837520123, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0069674295373260975, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06602954119443893, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6780332922935486, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.12804561853408813, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.06783650815486908, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.04643876105546951, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5977190136909485, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6034260988235474, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.45251166820526123, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.2501661777496338, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.20962758362293243, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8276961445808411, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.17588691413402557, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5750000476837158, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.583428680896759, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6471192836761475, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5253897905349731, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.30984044075012207, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.005415006540715694, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.382884979248047, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.16718576848506927, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.03674594312906265, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.4214879274368286, "metrics/particle/class_1/pid_accuracy": 0.8467749357223511, "metrics/particle/class_2/efficiency": 0.30682072043418884, "metrics/particle/class_2/pid_accuracy": 0.054113347083330154, "metrics/particle/class_3/efficiency": 0.4280779957771301, "metrics/particle/class_3/pid_accuracy": 0.8927533030509949, "metrics/particle/class_4/efficiency": 0.43407121300697327, "metrics/particle/class_4/pid_accuracy": 0.05543237179517746, "metrics/particle/class_5/efficiency": 0.47200000286102295, "metrics/particle/class_5/pid_accuracy": 0.09322033822536469, "metrics/particle/count/bias_mean": -6.62109375, "metrics/particle/count/mae": 9.05078125, "metrics/particle/count/prediction_mean": 63.43489456176758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.20321939885616302, "metrics/particle/event/energy_response_mean": 1.0294743776321411, "metrics/particle/event/met_abs_error": 17.173479080200195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1875811666250229, "metrics/particle/event/scalar_pt_response_mean": 1.0810329914093018, "metrics/particle/event/vector_pt_closure": 0.1343500167131424, "metrics/particle/matched/delta_eta_abs_mean": 0.029799863696098328, "metrics/particle/matched/delta_phi_abs_mean": 0.022330285981297493, "metrics/particle/matched/delta_r_mean": 0.04129164665937424, "metrics/particle/matched/energy_relative_abs_mean": 0.2065168023109436, "metrics/particle/matched/pid_accuracy": 0.8068176507949829, "metrics/particle/matched/pt_relative_abs_mean": 0.20363642275333405, "metrics/particle/matching/duplicate_fraction": 0.09675068408250809, "metrics/particle/matching/efficiency": 0.4151999056339264, "metrics/particle/matching/f1": 0.4357936382293701, "metrics/particle/matching/purity": 0.45853689312934875, "metrics/particle/matching/target_coverage_dr0p05": 0.32631081342697144, "metrics/particle/matching/target_coverage_dr0p10": 0.5095347762107849, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.2625039517879486, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5091165900230408, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.04035700857639313, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20617328584194183, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20446255803108215, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.012779898941516876, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24496014416217804, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0191256832331419, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.4210145473480225, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.1975812166929245, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.9857702255249023, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4392.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8587419390678406, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.448704242706299, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.629209518432617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 4.426192283630371, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.42735594511032104, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 23.217350006103516, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.044803448021411896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.4777112007141113, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.3417412340641022, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.8064093589782715, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9285.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4016093611717224, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 9.076480865478516, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 24.242610931396484, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 10.069913864135742, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.2186288833618164, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.3634750843048096, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 21.281158447265625, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -11.596847534179688, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 33.81988525390625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04345151036977768, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.21702204644680023, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.21162064373493195, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.006469665560871363, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.25260674953460693, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.06676557660102844, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.862349510192871, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.473337858915329, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.422573566436768, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 674.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.4957078695297241, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.6894314289093018, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8.073227882385254, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 7.895251750946045, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5402476787567139, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 30.210147857666016, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10732714086771011, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.559999942779541, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.6168646216392517, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.484416961669922, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1938.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.22310493886470795, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -16.766145706176758, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 20.853031158447266, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.9631779193878174, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3903335630893707, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.040374815464019775, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 20.509794235229492, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -6.7846832275390625, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 41.949344635009766, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.04163091257214546, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20778632164001465, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20439846813678741, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.010957193560898304, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24427519738674164, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002158717019483447, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.508233368396759, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0867813378572464, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2459417581558228, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 9728.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7037665247917175, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.508426189422607, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.828190326690674, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.631448268890381, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.41729840636253357, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.53081703186035, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.020380033180117607, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.3712793588638306, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.22636540234088898, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.786489486694336, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20314.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.45379483699798584, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 22.738292694091797, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 23.666088104248047, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 19.086856842041016, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.047929048538208, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 6.108134746551514, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.875658988952637, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -4.350323677062988, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 18.69675064086914, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.041335660964250565, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20772822201251984, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20474651455879211, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.011293865740299225, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24486418068408966, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.01013924553990364, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8864684104919434, "metrics/particle/momentum/all/accepted/pz_bias": -0.13728642463684082, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.237637519836426, "metrics/particle/momentum/all/accepted/samples": 14794.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.79805988073349, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.084646701812744, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.324887752532959, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4.127451419830322, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.42781496047973633, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 19.76887321472168, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.03291372209787369, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.892982840538025, "metrics/particle/momentum/all/assigned/pz_bias": -0.28433072566986084, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.221712112426758, "metrics/particle/momentum/all/assigned/samples": 31537.0, "metrics/particle/momentum/all/efficiency": 0.41798046231269836, "metrics/particle/momentum/all/event/energy_bias": 15.048625946044922, "metrics/particle/momentum/all/event/p3_error_mean": 50.99663543701172, "metrics/particle/momentum/all/event/pt_vector_error_mean": 22.558368682861328, "metrics/particle/momentum/all/event/px_bias": -2.656891345977783, "metrics/particle/momentum/all/event/py_bias": 7.5119853019714355, "metrics/particle/momentum/all/event/pz_abs_mean": 40.71097946166992, "metrics/particle/momentum/all/event/pz_bias": -22.731853485107422, "metrics/particle/momentum/all/event/pz_residual_rms": 55.75883102416992, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.05172828584909439, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2282322198152542, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.2235591858625412, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.012052944861352444, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2623012959957123, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.011636057868599892, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.0933085680007935, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.1813492327928543, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.447932004928589, "metrics/particle/momentum/class_1/accepted/samples": 7133.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.6692209243774414, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.7872962951660156, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.9723942279815674, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.751310348510742, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.4000000059604645, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 13.068941116333008, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.03547195345163345, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.9847174882888794, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.36534416675567627, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.837502479553223, "metrics/particle/momentum/class_1/assigned/samples": 14885.0, "metrics/particle/momentum/class_1/efficiency": 0.4274584949016571, "metrics/particle/momentum/class_1/event/energy_bias": 49.770511627197266, "metrics/particle/momentum/class_1/event/p3_error_mean": 42.33061981201172, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 21.133840560913086, "metrics/particle/momentum/class_1/event/px_bias": 1.7313199043273926, "metrics/particle/momentum/class_1/event/py_bias": 5.252866268157959, "metrics/particle/momentum/class_1/event/pz_abs_mean": 32.40751647949219, "metrics/particle/momentum/class_1/event/pz_bias": -21.183975219726562, "metrics/particle/momentum/class_1/event/pz_residual_rms": 42.92705535888672, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.047914452850818634, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2403031289577484, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23885752260684967, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08369582891464233, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2777419984340668, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03351955488324165, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8163342475891113, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11414611339569092, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.06544303894043, "metrics/particle/momentum/class_2/accepted/samples": 895.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9590165615081787, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8702719807624817, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.4375181198120117, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.0364458560943604, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39270687103271484, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 9.347670555114746, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06591865420341492, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0082175731658936, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19342681765556335, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.507639408111572, "metrics/particle/momentum/class_2/assigned/samples": 2139.0, "metrics/particle/momentum/class_2/efficiency": 0.30063822865486145, "metrics/particle/momentum/class_2/event/energy_bias": -42.87477111816406, "metrics/particle/momentum/class_2/event/p3_error_mean": 31.839866638183594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.192183494567871, "metrics/particle/momentum/class_2/event/px_bias": -0.4040735363960266, "metrics/particle/momentum/class_2/event/py_bias": -0.2627100944519043, "metrics/particle/momentum/class_2/event/pz_abs_mean": 24.092073440551758, "metrics/particle/momentum/class_2/event/pz_bias": -1.4685531854629517, "metrics/particle/momentum/class_2/event/pz_residual_rms": 37.29648208618164, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.028815515339374542, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18036113679409027, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17890143394470215, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0019995037000626326, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21848325431346893, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003369065932929516, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4783177673816681, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.07963761687278748, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3225862979888916, "metrics/particle/momentum/class_3/accepted/samples": 6530.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.9134884476661682, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.932923793792725, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 6.207956314086914, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 6.070087432861328, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.46387428045272827, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.988208770751953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.023965297266840935, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5974311828613281, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.20242390036582947, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.3590569496154785, "metrics/particle/momentum/class_3/assigned/samples": 14062.0, "metrics/particle/momentum/class_3/efficiency": 0.4294075071811676, "metrics/particle/momentum/class_3/event/energy_bias": 20.436565399169922, "metrics/particle/momentum/class_3/event/p3_error_mean": 28.14441680908203, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 15.893745422363281, "metrics/particle/momentum/class_3/event/px_bias": -2.9544928073883057, "metrics/particle/momentum/class_3/event/py_bias": 1.036617398262024, "metrics/particle/momentum/class_3/event/pz_abs_mean": 19.83592414855957, "metrics/particle/momentum/class_3/event/pz_bias": 0.23106440901756287, "metrics/particle/momentum/class_3/event/pz_residual_rms": 34.627925872802734, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04756159335374832, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.21603339910507202, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.21899323165416718, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.09188825637102127, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2523106634616852, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.07096774131059647, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.56354022026062, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.3851530849933624, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 4.858786106109619, "metrics/particle/momentum/class_4/accepted/samples": 155.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7848756313323975, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.004877090454102, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.197203636169434, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.9439778327941895, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4298780560493469, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.299938201904297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.057926829904317856, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.3773348331451416, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.46239039301872253, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.4305419921875, "metrics/particle/momentum/class_4/assigned/samples": 328.0, "metrics/particle/momentum/class_4/efficiency": 0.4258241653442383, "metrics/particle/momentum/class_4/event/energy_bias": -6.881922721862793, "metrics/particle/momentum/class_4/event/p3_error_mean": 6.226905822753906, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.533530235290527, "metrics/particle/momentum/class_4/event/px_bias": -0.3495663106441498, "metrics/particle/momentum/class_4/event/py_bias": 1.0607590675354004, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.529269218444824, "metrics/particle/momentum/class_4/event/pz_bias": 0.09166212379932404, "metrics/particle/momentum/class_4/event/pz_residual_rms": 10.786880493164062, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.05087830871343613, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23254616558551788, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22746390104293823, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06190255284309387, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2639605700969696, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0921154022216797, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6859019994735718, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.239982604980469, "metrics/particle/momentum/class_5/accepted/samples": 81.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.4293559789657593, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5586231350898743, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6023279428482056, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.14665183424949646, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.27642276883125305, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.4063888788223267, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10569106042385101, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.894789934158325, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.9504152536392212, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.322110176086426, "metrics/particle/momentum/class_5/assigned/samples": 123.0, "metrics/particle/momentum/class_5/efficiency": 0.5094339847564697, "metrics/particle/momentum/class_5/event/energy_bias": -5.401754856109619, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.079557418823242, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.978205919265747, "metrics/particle/momentum/class_5/event/px_bias": -0.6800787448883057, "metrics/particle/momentum/class_5/event/py_bias": 0.42445263266563416, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.5308125019073486, "metrics/particle/momentum/class_5/event/pz_bias": -0.4020545184612274, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.290961265563965, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.75412917137146, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7578003406524658, "metrics/set_slot/angular_recall_all_dr0p10": 0.8047041893005371, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5455272197723389, "metrics/set_slot/soft_count_bias_mean": -1.068817138671875}, "step_train_time": 4.191250562667847, "step_valid_time": 7177.781028747559, "step_total_time": 7181.972279787064, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1741529668403203, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.4089156604288525, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2981, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2326, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1928, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.768417575156921, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7802750754780275, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.774300932090546, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.21972492452197248, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6369342583415923, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6467628312646763, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6418109187749668, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.768417575156921, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1447248560047054, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3688864222182784, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2217, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1674, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1477, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7400530503978779, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7550744248985115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7474882786336235, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2449255751014885, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6529619805481874, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6662156066756879, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6595222147800849, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7400530503978779, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0657114997348465, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.34255938041486345, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1213, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 834, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7135093167701864, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7576257213520198, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.734906037584966, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.24237427864798022, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6475155279503105, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6875515251442704, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6669332267093163, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7135093167701864} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_30000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..e02edc04bc042a4ee3214bd92840fa2eea896b5e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4212620807276861, "train": {"Classification_binary": 0.18521030247211456, "Cardinality": 0.1318555325269699, "Classification": 0.09491446614265442, "Regression_pt": 0.19528235495090485, "Regression_eta": 0.0018253452144563198, "Regression_sin_phi": 0.0010717838304117322, "Regression_cos_phi": 0.0010567490244284272, "Regression_energy": 0.19352884590625763, "HitGrouping": 0.24017061293125153, "Auxiliary": 0.2775658071041107, "Total": 1.307600736618042, "Total_unweighted": 0.6728898475412279}, "train_group": {}, "valid": {"Classification_binary": 0.20796841382980347, "Cardinality": 0.1455246110757192, "Classification": 0.10315387447675069, "Regression_pt": 0.19765853881835938, "Regression_eta": 0.0020220810547471046, "Regression_sin_phi": 0.001099631345520417, "Regression_cos_phi": 0.001129190515105923, "Regression_energy": 0.1965588927268982, "HitGrouping": 0.23701101541519165, "Auxiliary": 0.29387350877126056, "Total": 1.3791146278381348, "Total_unweighted": 0.7095906227671852, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7345129251480103, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3355601727962494, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43033909797668457, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3262912929058075, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3357331454753876, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5994143486022949, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6843809485435486, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2879924178123474, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41274145245552063, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2772902548313141, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.31281235814094543, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6607929468154907, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7540702223777771, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.34175005555152893, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4072926342487335, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.33373162150382996, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3292039930820465, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5675550103187561, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7415260076522827, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.33493638038635254, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41519758105278015, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3262748122215271, "metrics/hit_grouping/calorimeter/all/object_purity": 0.32983478903770447, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5861701965332031, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02623826079070568, "metrics/hit_grouping/calorimeter/background_recall": 0.6351842284202576, "metrics/hit_grouping/calorimeter/completeness": 0.3853522837162018, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4096620976924896, "metrics/hit_grouping/calorimeter/energy_purity": 0.4049365818500519, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5045559406280518, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7611863613128662, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.35107535123825073, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3494177460670471, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.34407833218574524, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3513090908527374, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5356225967407227, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7344545125961304, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.34734171628952026, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4632459878921509, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.33699700236320496, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3402186930179596, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.608442485332489, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7072802186012268, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2878478467464447, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5043610334396362, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2774168848991394, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.27355968952178955, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6699532270431519, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7245994806289673, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3556080162525177, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5050762891769409, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3467830717563629, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3502108156681061, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6293366551399231, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7383877635002136, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4559009373188019, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.646258533000946, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.44293901324272156, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.42386794090270996, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6017791628837585, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7601246237754822, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.29723405838012695, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.28900495171546936, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2892983555793762, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2958964407444, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.53776615858078, "metrics/hit_grouping/calorimeter/purity": 0.37985169887542725, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.838647723197937, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06224890053272247, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14461538195610046, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06051281839609146, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.024619871750473976, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05846153944730759, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7570504546165466, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1334930807352066, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11060433089733124, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13194750249385834, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13036313652992249, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38255417346954346, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7385629415512085, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3523346483707428, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.43848374485969543, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3431161940097809, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3478129804134369, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6091290712356567, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7253214120864868, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4381350874900818, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.41378191113471985, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.40634360909461975, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6087865829467773, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7715943455696106, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3320246636867523, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4392518699169159, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4266066253185272, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48289400339126587, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.685350775718689, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4347483515739441, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37839534878730774, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3640460968017578, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6583445072174072, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7052476406097412, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42716795206069946, "metrics/hit_grouping/tracker/all/object_completeness": 0.39470526576042175, "metrics/hit_grouping/tracker/all/object_purity": 0.38268396258354187, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6279047727584839, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011218305677175522, "metrics/hit_grouping/tracker/background_recall": 0.05786886066198349, "metrics/hit_grouping/tracker/completeness": 0.4151375889778137, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5417864322662354, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7104578614234924, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42086559534072876, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.44756776094436646, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4437703788280487, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6297038793563843, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6969093680381775, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44907671213150024, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.37574729323387146, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3607631027698517, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6417285203933716, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.702541172504425, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41788026690483093, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2975006401538849, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2703714072704315, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6092233061790466, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7339422106742859, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45237240195274353, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4173719882965088, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3935546576976776, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5868080258369446, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7675758600234985, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47137150168418884, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4511823058128357, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3949236571788788, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5039947032928467, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6569066047668457, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3865605294704437, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3552456796169281, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3663700222969055, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7032310962677002, "metrics/hit_grouping/tracker/purity": 0.3982275128364563, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8779280185699463, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0152041707187891, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.023977132514119148, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02272525615990162, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08470895141363144, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6738455891609192, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.18455158174037933, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1763806939125061, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.17289213836193085, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5790565013885498, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6853588819503784, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.510772705078125, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.46994078159332275, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.45548444986343384, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7072781920433044, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4347827434539795, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5261197686195374, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5771857500076294, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5983074307441711, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.050944603979587555, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06750262528657913, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.9160213470459, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27777498960494995, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14733518660068512, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8207360506057739, "metrics/particle/class_1/pid_accuracy": 0.9549903273582458, "metrics/particle/class_2/efficiency": 0.38187122344970703, "metrics/particle/class_2/pid_accuracy": 0.5393654704093933, "metrics/particle/class_3/efficiency": 0.677364706993103, "metrics/particle/class_3/pid_accuracy": 0.9547728896141052, "metrics/particle/class_4/efficiency": 0.6602502465248108, "metrics/particle/class_4/pid_accuracy": 0.3090378940105438, "metrics/particle/class_5/efficiency": 0.8339999914169312, "metrics/particle/class_5/pid_accuracy": 0.3549160659313202, "metrics/particle/count/bias_mean": -2.9153645038604736, "metrics/particle/count/mae": 4.787760257720947, "metrics/particle/count/prediction_mean": 67.140625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08750507980585098, "metrics/particle/event/energy_response_mean": 0.9540013670921326, "metrics/particle/event/met_abs_error": 9.183148384094238, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07084383070468903, "metrics/particle/event/scalar_pt_response_mean": 0.9652915000915527, "metrics/particle/event/vector_pt_closure": 0.07000276446342468, "metrics/particle/matched/delta_eta_abs_mean": 0.016773449257016182, "metrics/particle/matched/delta_phi_abs_mean": 0.016677269712090492, "metrics/particle/matched/delta_r_mean": 0.02647222951054573, "metrics/particle/matched/energy_relative_abs_mean": 0.17235463857650757, "metrics/particle/matched/pid_accuracy": 0.927721381187439, "metrics/particle/matched/pt_relative_abs_mean": 0.16985048353672028, "metrics/particle/matching/duplicate_fraction": 0.07486812770366669, "metrics/particle/matching/efficiency": 0.7207869291305542, "metrics/particle/matching/f1": 0.736103355884552, "metrics/particle/matching/purity": 0.7520847916603088, "metrics/particle/matching/target_coverage_dr0p05": 0.6614222526550293, "metrics/particle/matching/target_coverage_dr0p10": 0.7898165583610535, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.48690593242645264, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7692972421646118, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025876399129629135, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17283043265342712, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17088760435581207, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0010619969107210636, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2114415019750595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010648087598383427, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.919342577457428, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.022811321541666985, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3244025707244873, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7607.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.36300620436668396, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5994012355804443, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7683731317520142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5661616325378418, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.19581730663776398, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.230215072631836, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02485942654311657, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.429055094718933, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12032439559698105, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.785043239593506, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10137.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6955925226211548, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.808593988418579, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.86379623413086, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.863044738769531, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5464811325073242, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.12547600269317627, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.30708122253418, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.24549251794815063, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.926233291625977, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0253542959690094, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1785302609205246, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1749768704175949, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006938912905752659, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21662811934947968, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.036795251071453094, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.819433569908142, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0744900181889534, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.390932559967041, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1685.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6014535427093506, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2166945934295654, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.178987979888916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.97628116607666, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2508143186569214, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.676273345947266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07125407457351685, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.101020097732544, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2338690310716629, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.782672882080078, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2456.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5577623248100281, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.734121322631836, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.423591613769531, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5066090822219849, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14922493696212769, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.014638196676969528, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.229424476623535, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.11673122644424438, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.76841926574707, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026468634605407715, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.170842245221138, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16830599308013916, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.004934302065521479, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20821326971054077, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008026673458516598, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2981497645378113, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0008671797695569694, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8750513792037964, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16196.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2731899917125702, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.569611668586731, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1920982599258423, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9853044152259827, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.17453621327877045, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.392794609069824, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006423469632863998, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5678896307945251, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.017474545165896416, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.126364231109619, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20861.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7555161714553833, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.595959424972534, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.503641128540039, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.768013000488281, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.050640821456909, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3716468811035156, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.134347438812256, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1938878297805786, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.60698127746582, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02621821127831936, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17194387316703796, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16951745748519897, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.002993663540109992, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2097461074590683, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006120527163147926, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5841187238693237, "metrics/particle/momentum/all/accepted/pz_bias": 0.012283673509955406, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8367571830749512, "metrics/particle/momentum/all/accepted/samples": 25488.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.32450467348098755, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5527292490005493, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5125830173492432, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3074777126312256, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18658456206321716, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.922430038452148, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016769295558333397, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0148019790649414, "metrics/particle/momentum/all/assigned/pz_bias": 0.04273252934217453, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5809779167175293, "metrics/particle/momentum/all/assigned/samples": 33454.0, "metrics/particle/momentum/all/efficiency": 0.7201220393180847, "metrics/particle/momentum/all/event/energy_bias": -12.138673782348633, "metrics/particle/momentum/all/event/p3_error_mean": 24.661537170410156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.261312484741211, "metrics/particle/momentum/all/event/px_bias": 2.447896957397461, "metrics/particle/momentum/all/event/py_bias": 0.2608090043067932, "metrics/particle/momentum/all/event/pz_abs_mean": 18.2659969329834, "metrics/particle/momentum/all/event/pz_bias": 0.3226490318775177, "metrics/particle/momentum/all/event/pz_residual_rms": 28.1762638092041, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028188757598400116, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17005814611911774, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1680133193731308, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.008030269294977188, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20711205899715424, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006838850677013397, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6486303210258484, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010802182368934155, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8902791738510132, "metrics/particle/momentum/class_1/accepted/samples": 13745.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16651301085948944, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.514440655708313, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.623306393623352, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.42914870381355286, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11765791475772858, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.404891014099121, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.012485392391681671, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.880393922328949, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008838470093905926, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7730672359466553, "metrics/particle/momentum/class_1/assigned/samples": 16259.0, "metrics/particle/momentum/class_1/efficiency": 0.8236950635910034, "metrics/particle/momentum/class_1/event/energy_bias": 2.610340118408203, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.079425811767578, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.287376403808594, "metrics/particle/momentum/class_1/event/px_bias": 0.2863653302192688, "metrics/particle/momentum/class_1/event/py_bias": 2.4960579872131348, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.17793083190918, "metrics/particle/momentum/class_1/event/pz_bias": -2.8194127082824707, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.31734275817871, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03640420362353325, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20919999480247498, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21133874356746674, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.044633448123931885, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24989168345928192, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028310501947999, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9174420833587646, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.15944704413414001, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.470483303070068, "metrics/particle/momentum/class_2/accepted/samples": 1095.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8625459671020508, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9355839490890503, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.084140658378601, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6188768744468689, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3773075044155121, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.728190898895264, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06573615223169327, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.101222515106201, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.4082559049129486, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.6483612060546875, "metrics/particle/momentum/class_2/assigned/samples": 2221.0, "metrics/particle/momentum/class_2/efficiency": 0.36781996488571167, "metrics/particle/momentum/class_2/event/energy_bias": -16.706829071044922, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.113548278808594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.923160552978516, "metrics/particle/momentum/class_2/event/px_bias": 1.5190259218215942, "metrics/particle/momentum/class_2/event/py_bias": -1.4060860872268677, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.602787017822266, "metrics/particle/momentum/class_2/event/pz_bias": 3.077829360961914, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.224367141723633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02248121239244938, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16951723396778107, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16613486409187317, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.00877378974109888, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20766904950141907, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0025338660925626755, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.32341820001602173, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.017861725762486458, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0643750429153442, "metrics/particle/momentum/class_3/accepted/samples": 10261.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4215867817401886, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.797884464263916, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.5670199394226074, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.389812469482422, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2341557741165161, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.878740310668945, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013546202331781387, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8186231255531311, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013406124897301197, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.3654675483703613, "metrics/particle/momentum/class_3/assigned/samples": 14469.0, "metrics/particle/momentum/class_3/efficiency": 0.6747550368309021, "metrics/particle/momentum/class_3/event/energy_bias": 5.638416767120361, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.601964950561523, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.589635372161865, "metrics/particle/momentum/class_3/event/px_bias": 0.41060250997543335, "metrics/particle/momentum/class_3/event/py_bias": -1.3848543167114258, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.64493465423584, "metrics/particle/momentum/class_3/event/pz_bias": -0.8368198871612549, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.22748374938965, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027865305542945862, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1962176114320755, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1920420229434967, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0018382990965619683, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2293664515018463, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007843137718737125, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1489953994750977, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.21268129348754883, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.30998158454895, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.34156548976898193, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.763411521911621, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.477391242980957, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.294520616531372, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24147726595401764, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.869873046875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017045455053448677, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2825957536697388, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2322392612695694, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.40700101852417, "metrics/particle/momentum/class_4/assigned/samples": 352.0, "metrics/particle/momentum/class_4/efficiency": 0.7005494236946106, "metrics/particle/momentum/class_4/event/energy_bias": -2.12576961517334, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.067294120788574, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1260623931884766, "metrics/particle/momentum/class_4/event/px_bias": 0.10282349586486816, "metrics/particle/momentum/class_4/event/py_bias": 0.21512015163898468, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8446073532104492, "metrics/particle/momentum/class_4/event/pz_bias": 0.44398531317710876, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.538748264312744, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02384237013757229, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2009887844324112, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19865263998508453, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.05719282105565071, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23666834831237793, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.022727273404598236, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9803531169891357, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.37465810775756836, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.898449420928955, "metrics/particle/momentum/class_5/accepted/samples": 132.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08340278267860413, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.310280978679657, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.29729989171028137, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.015966670587658882, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11764705926179886, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5282393097877502, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06535948067903519, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.9471240043640137, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.6758936643600464, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.300746917724609, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.8301886916160583, "metrics/particle/momentum/class_5/event/energy_bias": -1.5548334121704102, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0936527252197266, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2593765258789062, "metrics/particle/momentum/class_5/event/px_bias": 0.1290796548128128, "metrics/particle/momentum/class_5/event/py_bias": 0.3405713438987732, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7253056764602661, "metrics/particle/momentum/class_5/event/pz_bias": 0.4570671617984772, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.404604434967041, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4548529386520386, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4554695188999176, "metrics/set_slot/angular_recall_all_dr0p10": 0.9318718314170837, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7825121283531189, "metrics/set_slot/soft_count_bias_mean": 1.041804552078247}, "step_train_time": 5.382530450820923, "step_valid_time": 76338.79206609726, "step_total_time": 76344.17459702492, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9816161579854586, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18209896081916932, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2990, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2806, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2760, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9269904195573174, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9384615384615385, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9326907096559748, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06153846153846154, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9117938553022795, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9230769230769231, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9174006980222702, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9269904195573174, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9731980915496049, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16885744752717613, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2229, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2081, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2069, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9199823165340407, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.933602512337371, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9267423736361612, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06639748766262898, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9146772767462422, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9282189322566173, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9213983522600757, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9199823165340407, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9750879953489732, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17052794862690324, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1254, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1157, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1150, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8982919254658385, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9226475279106858, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9103068450039339, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0773524720893142, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8928571428571429, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9170653907496013, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9047993705743509, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8982919254658385} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_31000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..4e44f9a9ddd19d5be9df384c95ac0e5ba2a5b377 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4686374834186091, "train": {"Classification_binary": 0.1668454110622406, "Cardinality": 0.1343144029378891, "Classification": 0.09344537556171417, "Regression_pt": 0.1748921424150467, "Regression_eta": 0.0015324066625908017, "Regression_sin_phi": 0.0009895716793835163, "Regression_cos_phi": 0.0008329774718731642, "Regression_energy": 0.17278453707695007, "HitGrouping": 0.22541525959968567, "Auxiliary": 0.2547558844089508, "Total": 1.199539303779602, "Total_unweighted": 0.611322421929799}, "train_group": {}, "valid": {"Classification_binary": 0.2035603920618693, "Cardinality": 0.14253845810890198, "Classification": 0.09904571374257405, "Regression_pt": 0.1767974297205607, "Regression_eta": 0.0019466634839773178, "Regression_sin_phi": 0.0010108285738776128, "Regression_cos_phi": 0.0010177323905130227, "Regression_energy": 0.1758731802304586, "HitGrouping": 0.22972875833511353, "Auxiliary": 0.2752908269564311, "Total": 1.3050626118977864, "Total_unweighted": 0.6592519402038306, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.754457414150238, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3652826249599457, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41796544194221497, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3575790524482727, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.35552868247032166, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5647492408752441, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6893930435180664, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.33268481492996216, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42527955770492554, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3241766393184662, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34234827756881714, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6601152420043945, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7707944512367249, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36059191823005676, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4036773145198822, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35432302951812744, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.341314435005188, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5297489762306213, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7582358717918396, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.35959821939468384, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41021275520324707, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35266679525375366, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3459812104701996, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5527051687240601, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025873037055134773, "metrics/hit_grouping/calorimeter/background_recall": 0.6319948434829712, "metrics/hit_grouping/calorimeter/completeness": 0.413837194442749, "metrics/hit_grouping/calorimeter/energy_completeness": 0.434867262840271, "metrics/hit_grouping/calorimeter/energy_purity": 0.42976370453834534, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4978831708431244, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7804620862007141, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3751958906650543, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33789190649986267, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37000012397766113, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.36579135060310364, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.49943265318870544, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7524235844612122, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.37204885482788086, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.45305678248405457, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3640613555908203, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3598152995109558, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5742357969284058, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7172473073005676, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3136082887649536, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5186449289321899, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3041036128997803, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.28963324427604675, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6430286765098572, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7388510704040527, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3819182813167572, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4934288263320923, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3755893409252167, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.37199050188064575, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5978822708129883, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7346721887588501, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.47508031129837036, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6378859281539917, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4611864387989044, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.44998905062675476, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5907901525497437, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7820911407470703, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3208252787590027, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29161643981933594, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3141370415687561, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.30471041798591614, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4992634356021881, "metrics/hit_grouping/calorimeter/purity": 0.4077771008014679, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8429365158081055, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07810229808092117, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14615385234355927, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08001831918954849, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04413225129246712, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05999999865889549, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7764992117881775, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14016644656658173, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09692132472991943, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13737685978412628, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13122788071632385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.37571266293525696, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7553801536560059, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3782188892364502, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.433860182762146, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.37086063623428345, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.36476659774780273, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5733866691589355, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7219401597976685, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4187089204788208, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4312768280506134, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.44103583693504333, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6143156290054321, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7766462564468384, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35221537947654724, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47121840715408325, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46574071049690247, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.475602924823761, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6805697083473206, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43222808837890625, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3796691596508026, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3804278075695038, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6695282459259033, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7017370462417603, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4212166965007782, "metrics/hit_grouping/tracker/all/object_completeness": 0.40369096398353577, "metrics/hit_grouping/tracker/all/object_purity": 0.406769335269928, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6357453465461731, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.007805231958627701, "metrics/hit_grouping/tracker/background_recall": 0.04158028960227966, "metrics/hit_grouping/tracker/completeness": 0.4206911325454712, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5094436407089233, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7050496339797974, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4100227653980255, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4557447135448456, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47015875577926636, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6360819935798645, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6981469988822937, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4361317455768585, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38101083040237427, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.38365882635116577, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.643061101436615, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6981982588768005, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4302184581756592, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31223657727241516, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.29060834646224976, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6272249221801758, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7305765151977539, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4471603035926819, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42415761947631836, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.412316232919693, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5946261882781982, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7633642554283142, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4687083959579468, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46543562412261963, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43056756258010864, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5206391215324402, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6533305644989014, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37906813621520996, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.36617136001586914, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39535844326019287, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7095528244972229, "metrics/hit_grouping/tracker/purity": 0.4021666347980499, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8871857523918152, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.021720243617892265, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.039231907576322556, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03688570111989975, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0873153805732727, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6759704351425171, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19232763350009918, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1959153264760971, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20741751790046692, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5749092698097229, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6794716715812683, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5015642642974854, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4768734574317932, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.47972962260246277, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7171949744224548, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4116746187210083, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.560135006904602, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6239258646965027, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6060003638267517, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07405272126197815, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07519557327032089, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.96552276611328, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2905576527118683, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16011780500411987, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8297783732414246, "metrics/particle/class_1/pid_accuracy": 0.9620859622955322, "metrics/particle/class_2/efficiency": 0.39252859354019165, "metrics/particle/class_2/pid_accuracy": 0.5761646032333374, "metrics/particle/class_3/efficiency": 0.6807113885879517, "metrics/particle/class_3/pid_accuracy": 0.9556885957717896, "metrics/particle/class_4/efficiency": 0.6727622747421265, "metrics/particle/class_4/pid_accuracy": 0.3433476388454437, "metrics/particle/class_5/efficiency": 0.8360000252723694, "metrics/particle/class_5/pid_accuracy": 0.3732057511806488, "metrics/particle/count/bias_mean": -4.08984375, "metrics/particle/count/mae": 5.4921875, "metrics/particle/count/prediction_mean": 65.96614837646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08160268515348434, "metrics/particle/event/energy_response_mean": 0.9686405062675476, "metrics/particle/event/met_abs_error": 8.386176109313965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06573033332824707, "metrics/particle/event/scalar_pt_response_mean": 0.9886093735694885, "metrics/particle/event/vector_pt_closure": 0.06847671419382095, "metrics/particle/matched/delta_eta_abs_mean": 0.0175347737967968, "metrics/particle/matched/delta_phi_abs_mean": 0.015488585457205772, "metrics/particle/matched/delta_r_mean": 0.02619916759431362, "metrics/particle/matched/energy_relative_abs_mean": 0.16526587307453156, "metrics/particle/matched/pid_accuracy": 0.9335743188858032, "metrics/particle/matched/pt_relative_abs_mean": 0.16437266767024994, "metrics/particle/matching/duplicate_fraction": 0.06840432435274124, "metrics/particle/matching/efficiency": 0.7274966239929199, "metrics/particle/matching/f1": 0.7493705749511719, "metrics/particle/matching/purity": 0.7726007699966431, "metrics/particle/matching/target_coverage_dr0p05": 0.6697303056716919, "metrics/particle/matching/target_coverage_dr0p10": 0.7915543913841248, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5022024512290955, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7726799845695496, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024720899760723114, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16505186259746552, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1652384102344513, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014611705206334591, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2059164047241211, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009528715163469315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8639863729476929, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0010015829466283321, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2688193321228027, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7766.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.299981027841568, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0635334253311157, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2976187467575073, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1146721839904785, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16485178470611572, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.023575782775879, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01976204849779606, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.28550124168396, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.046910129487514496, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4806783199310303, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9918.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7101317048072815, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.4159523248672485, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.785462379455566, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.968001365661621, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.40679728984832764, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.0743423700332642, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.192605018615723, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.8913745880126953, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.184144973754883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022860581055283546, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1672223061323166, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16829028725624084, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0463242344558239, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21150706708431244, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029842013493180275, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6672459840774536, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.12991194427013397, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.391117095947266, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1709.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4968816041946411, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9959771037101746, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.683195114135742, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.512437105178833, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22005924582481384, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 22.12533950805664, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.061362676322460175, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7712900638580322, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2189578264951706, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.356202125549316, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2363.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.565706729888916, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.1730265617370605, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.397496223449707, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.586822509765625, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03870762139558792, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.058360159397125244, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.231792449951172, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.767061233520508, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.20815086364746, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02693912573158741, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16541258990764618, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16398859024047852, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011940193362534046, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20444898307323456, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008660686435177922, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2837752401828766, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02055530995130539, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.825497031211853, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16165.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.26666712760925293, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5304933786392212, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1291319131851196, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9344547390937805, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16650499403476715, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.187370300292969, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00562615180388093, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5475561618804932, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.037382856011390686, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.206865072250366, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20618.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7540700435638428, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.12323516607284546, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.911663055419922, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.209137916564941, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.10599865019321442, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.3118677139282227, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.8346452713012695, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.558641791343689, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.15078353881836, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02599540539085865, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16542395949363708, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16465386748313904, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015041179955005646, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20537151396274567, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005421217065304518, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5517265200614929, "metrics/particle/momentum/all/accepted/pz_bias": -0.004603562410920858, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8094067573547363, "metrics/particle/momentum/all/accepted/samples": 25640.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2932455837726593, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3513277769088745, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.291547179222107, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.102124571800232, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16985318064689636, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.473533630371094, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013890999369323254, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9297447204589844, "metrics/particle/momentum/all/assigned/pz_bias": -0.053296804428100586, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.423428773880005, "metrics/particle/momentum/all/assigned/samples": 32899.0, "metrics/particle/momentum/all/efficiency": 0.7244165539741516, "metrics/particle/momentum/all/event/energy_bias": -8.712214469909668, "metrics/particle/momentum/all/event/p3_error_mean": 25.03261375427246, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.132162094116211, "metrics/particle/momentum/all/event/px_bias": -0.3395063281059265, "metrics/particle/momentum/all/event/py_bias": -2.444570541381836, "metrics/particle/momentum/all/event/pz_abs_mean": 18.772371292114258, "metrics/particle/momentum/all/event/pz_bias": -6.217077732086182, "metrics/particle/momentum/all/event/pz_residual_rms": 29.576194763183594, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027774963527917862, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1607038825750351, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16011527180671692, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.023557502776384354, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20006616413593292, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00577034056186676, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6047549843788147, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.022582825273275375, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7810659408569336, "metrics/particle/momentum/class_1/accepted/samples": 13864.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.15408337116241455, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.41929855942726135, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5028695464134216, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3407410681247711, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10729958862066269, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.691469192504883, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009703936986625195, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7952351570129395, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03312951326370239, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.784391164779663, "metrics/particle/momentum/class_1/assigned/samples": 16179.0, "metrics/particle/momentum/class_1/efficiency": 0.8308264017105103, "metrics/particle/momentum/class_1/event/energy_bias": 12.86489200592041, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.488868713378906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.79117202758789, "metrics/particle/momentum/class_1/event/px_bias": -0.43612009286880493, "metrics/particle/momentum/class_1/event/py_bias": -0.8222455978393555, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.9308500289917, "metrics/particle/momentum/class_1/event/pz_bias": -2.476827621459961, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.132429122924805, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03537295386195183, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20179353654384613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20239852368831635, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03604545071721077, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2425125688314438, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026292726397514343, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.712773084640503, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.17976510524749756, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9915106296539307, "metrics/particle/momentum/class_2/accepted/samples": 1141.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7212217450141907, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9273144602775574, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.3865114450454712, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.9562767744064331, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34780555963516235, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 14.689138412475586, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05663048475980759, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7569565773010254, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10789994895458221, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.443910121917725, "metrics/particle/momentum/class_2/assigned/samples": 2119.0, "metrics/particle/momentum/class_2/efficiency": 0.3832717537879944, "metrics/particle/momentum/class_2/event/energy_bias": -19.395084381103516, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.18950080871582, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.094427108764648, "metrics/particle/momentum/class_2/event/px_bias": 0.7449117302894592, "metrics/particle/momentum/class_2/event/py_bias": -1.025771975517273, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.970907211303711, "metrics/particle/momentum/class_2/event/pz_bias": -2.3524603843688965, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.5155029296875, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022600507363677025, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16759487986564636, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16657540202140808, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.009160472080111504, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20778091251850128, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020479813683778048, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.31913575530052185, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007244087755680084, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3253276348114014, "metrics/particle/momentum/class_3/accepted/samples": 10254.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.38929763436317444, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4183521270751953, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.1611740589141846, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.975738286972046, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21361702680587769, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.258938789367676, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011063829995691776, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.774689257144928, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.060477565973997116, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.38071870803833, "metrics/particle/momentum/class_3/assigned/samples": 14100.0, "metrics/particle/momentum/class_3/efficiency": 0.6742947101593018, "metrics/particle/momentum/class_3/event/energy_bias": 0.9713948369026184, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.364657402038574, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.274591445922852, "metrics/particle/momentum/class_3/event/px_bias": -0.39400196075439453, "metrics/particle/momentum/class_3/event/py_bias": -1.1786491870880127, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.683161735534668, "metrics/particle/momentum/class_3/event/pz_bias": -1.5734585523605347, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.656997680664062, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028421534225344658, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17295637726783752, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16377529501914978, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03868234530091286, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21051529049873352, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011952191591262817, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1127711534500122, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.017898187041282654, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6937403678894043, "metrics/particle/momentum/class_4/accepted/samples": 251.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.365495502948761, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.512356281280518, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.5873982906341553, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.4435126781463623, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23188406229019165, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.873708724975586, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03188405930995941, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4933785200119019, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.15247517824172974, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.359236240386963, "metrics/particle/momentum/class_4/assigned/samples": 345.0, "metrics/particle/momentum/class_4/efficiency": 0.6895604133605957, "metrics/particle/momentum/class_4/event/energy_bias": -1.6355245113372803, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.189028024673462, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1687803268432617, "metrics/particle/momentum/class_4/event/px_bias": -0.1024758592247963, "metrics/particle/momentum/class_4/event/py_bias": 0.2176726758480072, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9484801292419434, "metrics/particle/momentum/class_4/event/pz_bias": 0.32147616147994995, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.760272026062012, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017001641914248466, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1638132929801941, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16752596199512482, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.02660217694938183, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20717598497867584, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03846153989434242, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9688318967819214, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6142115592956543, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.8767971992492676, "metrics/particle/momentum/class_5/accepted/samples": 130.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07121625542640686, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3396955132484436, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.32987862825393677, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11969418078660965, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14743590354919434, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1488412618637085, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0833333358168602, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.828451633453369, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5348182916641235, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.335429668426514, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8176100850105286, "metrics/particle/momentum/class_5/event/energy_bias": -1.5178905725479126, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.937765121459961, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2624335289001465, "metrics/particle/momentum/class_5/event/px_bias": -0.15182006359100342, "metrics/particle/momentum/class_5/event/py_bias": 0.3644236922264099, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.525114893913269, "metrics/particle/momentum/class_5/event/pz_bias": -0.1358068287372589, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.670583724975586, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4377463161945343, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.44023436307907104, "metrics/set_slot/angular_recall_all_dr0p10": 0.9383212924003601, "metrics/set_slot/angular_recall_selected_dr0p10": 0.783887505531311, "metrics/set_slot/soft_count_bias_mean": 0.7116899490356445}, "step_train_time": 4.949808359146118, "step_valid_time": 78795.833122015, "step_total_time": 78800.78293085098, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0080348771159184, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1756408115419611, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3011, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2833, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2798, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9359101420548398, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408834274327466, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.93839019542895, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0591165725672534, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9243475388173109, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9292593822650282, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9267969526333223, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9359101420548398, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9956892345714355, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17792887871126262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2108, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372236958443855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9443207126948775, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9407588196139339, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0556792873051225, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319186560565871, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9389755011135857, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354337696915909, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372236958443855, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9975763961382764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17243146095712247, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1171, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1167, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9091614906832298, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9286280729579699, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9187916830129462, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07137192704203013, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.906055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9254559873116575, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9156531973322871, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9091614906832298} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_32000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..d0af8bdcf0bb0f311833c15bea922d7d78158ad0 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.5160128861095319, "train": {"Classification_binary": 0.17128708958625793, "Cardinality": 0.12142150849103928, "Classification": 0.08901555091142654, "Regression_pt": 0.17733998596668243, "Regression_eta": 0.0016526258550584316, "Regression_sin_phi": 0.0009569870308041573, "Regression_cos_phi": 0.0009603830403648317, "Regression_energy": 0.17678022384643555, "HitGrouping": 0.22532440721988678, "Auxiliary": 0.2585347592830658, "Total": 1.2067309617996216, "Total_unweighted": 0.6179928462370299}, "train_group": {}, "valid": {"Classification_binary": 0.19230125347773233, "Cardinality": 0.1463760236899058, "Classification": 0.09808958570162456, "Regression_pt": 0.17403292655944824, "Regression_eta": 0.0018009102592865627, "Regression_sin_phi": 0.0010220314531276624, "Regression_cos_phi": 0.0010655030297736328, "Regression_energy": 0.17284039656321207, "HitGrouping": 0.22590829928716025, "Auxiliary": 0.2716331481933594, "Total": 1.2862943808237712, "Total_unweighted": 0.6411526070442051, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7506062388420105, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3700092136859894, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4273165166378021, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3634994626045227, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3673171401023865, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5733446478843689, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6914767622947693, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.32766303420066833, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4069806933403015, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.31648698449134827, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3428710699081421, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6475771069526672, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.775843620300293, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3777906000614166, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4057948589324951, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3702618181705475, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3556142747402191, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.524480938911438, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7601550221443176, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.37079039216041565, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4128267467021942, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3632623553276062, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3582371473312378, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5512462258338928, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02592899650335312, "metrics/hit_grouping/calorimeter/background_recall": 0.6320688724517822, "metrics/hit_grouping/calorimeter/completeness": 0.42451563477516174, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44772008061408997, "metrics/hit_grouping/calorimeter/energy_purity": 0.4425196647644043, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4907270669937134, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7809131741523743, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3813285827636719, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33771276473999023, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3767803907394409, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37512335181236267, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.49961182475090027, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7550866603851318, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.39142128825187683, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.45972830057144165, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3812559247016907, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.375573068857193, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5687772631645203, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7214985489845276, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.32698503136634827, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5229427218437195, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3158980906009674, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.30442875623703003, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6422702670097351, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7472951412200928, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.40263858437538147, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5002802610397339, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.39454197883605957, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3881292939186096, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5864216685295105, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7472182512283325, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.47569456696510315, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.633176326751709, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.46505093574523926, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4607110917568207, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5766614079475403, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7756357789039612, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3231276571750641, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29061201214790344, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.31660956144332886, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3129882514476776, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5101780891418457, "metrics/hit_grouping/calorimeter/purity": 0.41832318902015686, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8385207653045654, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07678990066051483, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16307692229747772, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07378388196229935, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.029498469084501266, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0615384615957737, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7750107049942017, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14445063471794128, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10946407914161682, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14150065183639526, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13733415305614471, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38084378838539124, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7576302886009216, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3900749087333679, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.43559810519218445, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3821871280670166, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37794938683509827, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5714848041534424, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7263128161430359, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43081292510032654, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4408113658428192, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4459612965583801, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6078900098800659, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7769376039505005, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3376331925392151, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4595710039138794, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4508366584777832, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.475602924823761, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6810253262519836, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4414428472518921, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39581409096717834, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3963819146156311, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6696069836616516, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7034169435501099, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4293406307697296, "metrics/hit_grouping/tracker/all/object_completeness": 0.4154060482978821, "metrics/hit_grouping/tracker/all/object_purity": 0.4166387617588043, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6337615847587585, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.007698478177189827, "metrics/hit_grouping/tracker/background_recall": 0.040751051157712936, "metrics/hit_grouping/tracker/completeness": 0.4322483539581299, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5233187675476074, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7116254568099976, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41111618280410767, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.464277446269989, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4753618836402893, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6303417086601257, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6973406672477722, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4544070065021515, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3991217315196991, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3917466700077057, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6411574482917786, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6916514039039612, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4431634247303009, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3242202401161194, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31272944808006287, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6334951519966125, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7304054498672485, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4572250247001648, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44216662645339966, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4326780140399933, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5949856042861938, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7537317872047424, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4653795063495636, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45336446166038513, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4319040775299072, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5439414381980896, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6594108939170837, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3866775929927826, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3738693594932556, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3930596113204956, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7000702619552612, "metrics/hit_grouping/tracker/purity": 0.4131705164909363, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8912206292152405, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02128583751618862, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03307051211595535, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03160644322633743, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08123371005058289, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6833084225654602, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20943494141101837, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22084037959575653, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22239995002746582, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5619491934776306, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6801872849464417, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5098282098770142, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4895133972167969, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.49107542634010315, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7170178890228271, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4259336590766907, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.579666793346405, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6428601741790771, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5964456796646118, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05979369580745697, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0656408742070198, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.9464168548584, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2933420240879059, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16290220618247986, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8480807542800903, "metrics/particle/class_1/pid_accuracy": 0.9626008868217468, "metrics/particle/class_2/efficiency": 0.37985190749168396, "metrics/particle/class_2/pid_accuracy": 0.5800354480743408, "metrics/particle/class_3/efficiency": 0.6839079260826111, "metrics/particle/class_3/pid_accuracy": 0.9595031142234802, "metrics/particle/class_4/efficiency": 0.6891241669654846, "metrics/particle/class_4/pid_accuracy": 0.3463687002658844, "metrics/particle/class_5/efficiency": 0.8619999885559082, "metrics/particle/class_5/pid_accuracy": 0.4106728434562683, "metrics/particle/count/bias_mean": -4.134765625, "metrics/particle/count/mae": 5.233724117279053, "metrics/particle/count/prediction_mean": 65.92122650146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08580460399389267, "metrics/particle/event/energy_response_mean": 0.9702957272529602, "metrics/particle/event/met_abs_error": 8.564492225646973, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06857062131166458, "metrics/particle/event/scalar_pt_response_mean": 0.9928755760192871, "metrics/particle/event/vector_pt_closure": 0.06778135150671005, "metrics/particle/matched/delta_eta_abs_mean": 0.014539132826030254, "metrics/particle/matched/delta_phi_abs_mean": 0.015571840107440948, "metrics/particle/matched/delta_r_mean": 0.023852812126278877, "metrics/particle/matched/energy_relative_abs_mean": 0.16173215210437775, "metrics/particle/matched/pid_accuracy": 0.9364482760429382, "metrics/particle/matched/pt_relative_abs_mean": 0.16051256656646729, "metrics/particle/matching/duplicate_fraction": 0.06370055675506592, "metrics/particle/matching/efficiency": 0.7367061376571655, "metrics/particle/matching/f1": 0.7591077089309692, "metrics/particle/matching/purity": 0.782914400100708, "metrics/particle/matching/target_coverage_dr0p05": 0.6893110275268555, "metrics/particle/matching/target_coverage_dr0p10": 0.7998253107070923, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5173503160476685, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7792409062385559, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022869378328323364, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16263015568256378, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16221876442432404, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016003316268324852, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20265261828899384, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010249271988868713, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8657301664352417, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0010543551761657, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.305635929107666, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7903.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2822938859462738, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1676726341247559, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3747328519821167, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1939741373062134, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16040578484535217, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.240632057189941, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02028927206993103, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2566524744033813, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.003609160427004099, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3487348556518555, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9956.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7226591110229492, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.5540679693222046, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.50183391571045, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.643704891204834, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1260310709476471, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.18430450558662415, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.083483695983887, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4773687422275543, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.37958526611328, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023114312440156937, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17039074003696442, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.167728453874588, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006845447234809399, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20797570049762726, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.031126203015446663, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.701371192932129, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.10690506547689438, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.290193557739258, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1767.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4563319683074951, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0385541915893555, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3168413639068604, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1229469776153564, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20100712776184082, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.0770320892334, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06420478224754333, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.752937078475952, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.37540656328201294, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.730910778045654, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2383.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5849056839942932, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.124213218688965, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.70709228515625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7232401371002197, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.18570727109909058, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.21830317378044128, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.507646560668945, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.1728217601776123, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.050765991210938, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024051040410995483, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1597907394170761, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15897957980632782, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021200867369771004, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20009465515613556, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007910911226645112, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2707347869873047, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0040397015400230885, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7891305685043335, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16433.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2511849105358124, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3033097982406616, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0398194789886475, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8579453825950623, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.160478875041008, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.144341468811035, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006107018329203129, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5270461440086365, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012438543140888214, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1772732734680176, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20632.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7665718197822571, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.2095609903335571, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.0299654006958, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.401885986328125, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6197154521942139, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.256124496459961, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.841488838195801, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.258658766746521, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.057546615600586, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023629870265722275, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1613679677248001, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16055253148078918, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018655478954315186, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20141370594501495, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005708156153559685, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5477215051651001, "metrics/particle/momentum/all/accepted/pz_bias": -0.009460716508328915, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.802069067955017, "metrics/particle/momentum/all/accepted/samples": 26103.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27540579438209534, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2432169914245605, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2332483530044556, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.05084228515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1633860021829605, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.127487182617188, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014588577672839165, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9082374572753906, "metrics/particle/momentum/all/assigned/pz_bias": -0.01825934834778309, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.266855478286743, "metrics/particle/momentum/all/assigned/samples": 32971.0, "metrics/particle/momentum/all/efficiency": 0.737497866153717, "metrics/particle/momentum/all/event/energy_bias": -8.468720436096191, "metrics/particle/momentum/all/event/p3_error_mean": 24.179224014282227, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.8876371383667, "metrics/particle/momentum/all/event/px_bias": 0.30797716975212097, "metrics/particle/momentum/all/event/py_bias": 0.8535167574882507, "metrics/particle/momentum/all/event/pz_abs_mean": 18.02911376953125, "metrics/particle/momentum/all/event/pz_bias": -3.908849000930786, "metrics/particle/momentum/all/event/pz_residual_rms": 28.881519317626953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02502533048391342, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15223081409931183, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1516706794500351, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.018747808411717415, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19117671251296997, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006059751845896244, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5830462574958801, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02118751034140587, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.77359938621521, "metrics/particle/momentum/class_1/accepted/samples": 14192.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13527712225914001, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3814265727996826, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.42813748121261597, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2710244953632355, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09166152030229568, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6741597652435303, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010623841546475887, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7485353350639343, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02649386040866375, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.298884153366089, "metrics/particle/momentum/class_1/assigned/samples": 16190.0, "metrics/particle/momentum/class_1/efficiency": 0.8504824042320251, "metrics/particle/momentum/class_1/event/energy_bias": 6.6315999031066895, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.371978759765625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.99645709991455, "metrics/particle/momentum/class_1/event/px_bias": -0.6548560857772827, "metrics/particle/momentum/class_1/event/py_bias": -0.8328982591629028, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.379319190979004, "metrics/particle/momentum/class_1/event/pz_bias": -3.115617275238037, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.693021774291992, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03236730769276619, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20260901749134064, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20558032393455505, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.014624841511249542, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24518413841724396, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.027100270614027977, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8584873676300049, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1777622103691101, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.309575080871582, "metrics/particle/momentum/class_2/accepted/samples": 1107.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7292841076850891, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8182023763656616, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8912067413330078, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4742870032787323, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35333654284477234, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.7220029830932617, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.053288526833057404, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.80551815032959, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05856618285179138, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.874676704406738, "metrics/particle/momentum/class_2/assigned/samples": 2083.0, "metrics/particle/momentum/class_2/efficiency": 0.371850848197937, "metrics/particle/momentum/class_2/event/energy_bias": -16.75687026977539, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.927082061767578, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.771162033081055, "metrics/particle/momentum/class_2/event/px_bias": 0.8282002806663513, "metrics/particle/momentum/class_2/event/py_bias": -0.3277814984321594, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.883796691894531, "metrics/particle/momentum/class_2/event/pz_bias": -0.3810344636440277, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.426008224487305, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020787373185157776, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1686110496520996, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16729199886322021, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020992616191506386, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20895755290985107, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002784712938591838, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.33090469241142273, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.011223400942981243, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2739722728729248, "metrics/particle/momentum/class_3/accepted/samples": 10414.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3688802719116211, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.2370028495788574, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.1626791954040527, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9849414825439453, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21635326743125916, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.080820083618164, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012536094523966312, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7768297791481018, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.01891903020441532, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.3116519451141357, "metrics/particle/momentum/class_3/assigned/samples": 14199.0, "metrics/particle/momentum/class_3/efficiency": 0.6848161816596985, "metrics/particle/momentum/class_3/event/energy_bias": 3.9484610557556152, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.591571807861328, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.967763900756836, "metrics/particle/momentum/class_3/event/px_bias": 0.12892411649227142, "metrics/particle/momentum/class_3/event/py_bias": 1.3538780212402344, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.066659927368164, "metrics/particle/momentum/class_3/event/pz_bias": -0.4813598394393921, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.944936752319336, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026925383135676384, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17251171171665192, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16417820751667023, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06842053681612015, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21032191812992096, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0117647061124444, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0074868202209473, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13877274096012115, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1093294620513916, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3574630320072174, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.7750661373138428, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.255326986312866, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.131472110748291, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21865889430046082, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.81803321838379, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02332361601293087, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4387675523757935, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.27048009634017944, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.569682598114014, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.7005494236946106, "metrics/particle/momentum/class_4/event/energy_bias": -1.1485177278518677, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.1390111446380615, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.188779830932617, "metrics/particle/momentum/class_4/event/px_bias": 0.07009527087211609, "metrics/particle/momentum/class_4/event/py_bias": 0.11201005429029465, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8863587379455566, "metrics/particle/momentum/class_4/event/pz_bias": 0.2216206192970276, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.007025718688965, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018330790102481842, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2039540410041809, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19830210506916046, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007558617740869522, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2388152927160263, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.007407407276332378, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.942857265472412, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.4559204876422882, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3652915954589844, "metrics/particle/momentum/class_5/accepted/samples": 135.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0694078877568245, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.33618852496147156, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3143564462661743, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08471642434597015, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.12820513546466827, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.584694504737854, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.943028450012207, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7642953395843506, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.654218673706055, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.849056601524353, "metrics/particle/momentum/class_5/event/energy_bias": -1.1433954238891602, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0186686515808105, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2845189571380615, "metrics/particle/momentum/class_5/event/px_bias": -0.06438644975423813, "metrics/particle/momentum/class_5/event/py_bias": 0.5483084321022034, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6163440942764282, "metrics/particle/momentum/class_5/event/pz_bias": -0.15245798230171204, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.801365852355957, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41393205523490906, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4149574637413025, "metrics/set_slot/angular_recall_all_dr0p10": 0.9381075501441956, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7941100001335144, "metrics/set_slot/soft_count_bias_mean": -0.9253033995628357}, "step_train_time": 4.810261964797974, "step_valid_time": 81591.6986515522, "step_total_time": 81596.50891399384, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006278824282155, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17225163980209746, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2844, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2807, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9395441030723488, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9451645064805583, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9423459244532804, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054835493519441676, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9273207796498183, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9328680624792289, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9300861497680584, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9395441030723488, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9935887830520523, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17056644367698193, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2113, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2099, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9341290893015031, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9428826416778224, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9384854541416833, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0571173583221776, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9279398762157383, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366354306113343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9322673773040195, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9341290893015031, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004147099717839, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1800134250493225, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1263, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1165, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9130434782608695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9311163895486936, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9219913759310074, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0688836104513064, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9045031055900621, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9224069675376089, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9133673069384555, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9130434782608695} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_33000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..b5cd5a747299ac5b4a0c5119fb4b2df1cd2350e2 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5633882888004549, "train": {"Classification_binary": 0.16503497958183289, "Cardinality": 0.13650929927825928, "Classification": 0.0832214280962944, "Regression_pt": 0.1709851175546646, "Regression_eta": 0.0015798585955053568, "Regression_sin_phi": 0.0009320302633568645, "Regression_cos_phi": 0.0010027626994997263, "Regression_energy": 0.17064054310321808, "HitGrouping": 0.22668513655662537, "Auxiliary": 0.2553103566169739, "Total": 1.2029969692230225, "Total_unweighted": 0.5933967198943719}, "train_group": {}, "valid": {"Classification_binary": 0.19364988803863525, "Cardinality": 0.14971913894017538, "Classification": 0.09954024354616801, "Regression_pt": 0.1782375971476237, "Regression_eta": 0.0018567478594680626, "Regression_sin_phi": 0.0010250029154121876, "Regression_cos_phi": 0.0010822043598939974, "Regression_energy": 0.17704119284947714, "HitGrouping": 0.2270601987838745, "Auxiliary": 0.2773035566012065, "Total": 1.3062196572621663, "Total_unweighted": 0.6524328767166784, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7535885572433472, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3692162334918976, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42363277077674866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3619205355644226, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.35864362120628357, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5676773190498352, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7006071209907532, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3239142894744873, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.40562522411346436, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3161740303039551, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34712734818458557, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6191121935844421, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7716629505157471, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36965498328208923, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.399029016494751, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3618597686290741, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3491983115673065, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5334159731864929, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7594732642173767, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.36541104316711426, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4075379967689514, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3577814996242523, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3520519733428955, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5521276593208313, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025974061340093613, "metrics/hit_grouping/calorimeter/background_recall": 0.6349930167198181, "metrics/hit_grouping/calorimeter/completeness": 0.41833946108818054, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4414561688899994, "metrics/hit_grouping/calorimeter/energy_purity": 0.43632349371910095, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4877830445766449, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7787618637084961, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.379894882440567, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33424901962280273, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37558311223983765, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3729437589645386, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5009256601333618, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7587535381317139, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.38363221287727356, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4537845849990845, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3731796443462372, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.367549329996109, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5587093830108643, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7193961143493652, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.31576529145240784, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5144734978675842, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.30405500531196594, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29168128967285156, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6536468267440796, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7459462881088257, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3953775465488434, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49579569697380066, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3872762620449066, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3758010268211365, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5869199633598328, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7388193607330322, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.471232533454895, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.633176326751709, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4567486345767975, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.44339442253112793, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5954997539520264, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7766586542129517, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.31965386867523193, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2837819755077362, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.31340861320495605, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3148317337036133, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5091736912727356, "metrics/hit_grouping/calorimeter/purity": 0.4122561812400818, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.847609281539917, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06487976014614105, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16769230365753174, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06434065848588943, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.033410198986530304, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05076923221349716, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7721587419509888, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14979618787765503, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1220068410038948, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.146654412150383, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1327698975801468, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3836944103240967, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7568651437759399, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38421788811683655, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.42907267808914185, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3761790990829468, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37145575881004333, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5725013017654419, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7275130152702332, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43693962693214417, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4436248242855072, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4417742192745209, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6066945791244507, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7842210531234741, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32641616463661194, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4672433137893677, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4673674404621124, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45709478855133057, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6662078499794006, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4444356858730316, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38301408290863037, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.38814055919647217, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6856737732887268, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6955235004425049, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43212732672691345, "metrics/hit_grouping/tracker/all/object_completeness": 0.4092651903629303, "metrics/hit_grouping/tracker/all/object_purity": 0.41176503896713257, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6414604187011719, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00896427407860756, "metrics/hit_grouping/tracker/background_recall": 0.046141088008880615, "metrics/hit_grouping/tracker/completeness": 0.42740872502326965, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5192871689796448, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7084425091743469, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4196811020374298, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4644542932510376, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4728375971317291, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6317084431648254, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.688949465751648, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4530744254589081, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3922008275985718, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.390891969203949, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6461069583892822, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.673829972743988, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4375, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30488377809524536, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2983701229095459, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.658171534538269, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7191330790519714, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4560568034648895, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4298674464225769, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.42454254627227783, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6069374680519104, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7179034948348999, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46804261207580566, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.43570676445961, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4298141300678253, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5965379476547241, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.660831093788147, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3946382701396942, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3777763545513153, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39194563031196594, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6943338513374329, "metrics/hit_grouping/tracker/purity": 0.4090662896633148, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8867282867431641, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01824500411748886, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04069489240646362, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.040658414363861084, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08688097447156906, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.677479088306427, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2218766212463379, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21782077848911285, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.21638375520706177, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5681700110435486, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6715967059135437, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5123074054718018, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4811466932296753, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.48443955183029175, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7251638174057007, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3915877938270569, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5783306360244751, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6420167684555054, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6087049841880798, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.09413952380418777, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0779002234339714, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.2656307220459, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2783464789390564, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14790664613246918, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8415116667747498, "metrics/particle/class_1/pid_accuracy": 0.9584999084472656, "metrics/particle/class_2/efficiency": 0.3934260606765747, "metrics/particle/class_2/pid_accuracy": 0.6073567271232605, "metrics/particle/class_3/efficiency": 0.6820629239082336, "metrics/particle/class_3/pid_accuracy": 0.9573176503181458, "metrics/particle/class_4/efficiency": 0.6823869347572327, "metrics/particle/class_4/pid_accuracy": 0.3187588155269623, "metrics/particle/class_5/efficiency": 0.8600000143051147, "metrics/particle/class_5/pid_accuracy": 0.40697672963142395, "metrics/particle/count/bias_mean": -4.651041507720947, "metrics/particle/count/mae": 5.6171875, "metrics/particle/count/prediction_mean": 65.40494537353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08014746755361557, "metrics/particle/event/energy_response_mean": 0.975638210773468, "metrics/particle/event/met_abs_error": 8.41395092010498, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0654529258608818, "metrics/particle/event/scalar_pt_response_mean": 0.9948431849479675, "metrics/particle/event/vector_pt_closure": 0.06666679680347443, "metrics/particle/matched/delta_eta_abs_mean": 0.016142824664711952, "metrics/particle/matched/delta_phi_abs_mean": 0.015692541375756264, "metrics/particle/matched/delta_r_mean": 0.025193320587277412, "metrics/particle/matched/energy_relative_abs_mean": 0.15983791649341583, "metrics/particle/matched/pid_accuracy": 0.9336828589439392, "metrics/particle/matched/pt_relative_abs_mean": 0.16000330448150635, "metrics/particle/matching/duplicate_fraction": 0.0665624812245369, "metrics/particle/matching/efficiency": 0.7338716983795166, "metrics/particle/matching/f1": 0.7590691447257996, "metrics/particle/matching/purity": 0.7860584259033203, "metrics/particle/matching/target_coverage_dr0p05": 0.6827686429023743, "metrics/particle/matching/target_coverage_dr0p10": 0.7954760789871216, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5164767503738403, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.775337815284729, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02432934381067753, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15699896216392517, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15747392177581787, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007636054418981075, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19834166765213013, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011198778636753559, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8421767354011536, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07465411722660065, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2045340538024902, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7858.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2935600280761719, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.425504207611084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5703797340393066, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3944883346557617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15714576840400696, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.712974548339844, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021345801651477814, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2694098949432373, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.057222891598939896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4829556941986084, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9838.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7185442447662354, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.4691543579101562, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.029701232910156, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.510262966156006, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.18337678909301758, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2350323647260666, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.583963394165039, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.90157151222229, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.7838134765625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024124979972839355, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1614716500043869, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1631058305501938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025370139628648758, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2026655226945877, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027475671842694283, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6545881032943726, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06844519078731537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.255733966827393, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1747.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.45303890109062195, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0328965187072754, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.414294958114624, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2466254234313965, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19736842811107635, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.26203155517578, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05814940482378006, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6727302074432373, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19298112392425537, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.790536403656006, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2356.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5782853364944458, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.5378828048706055, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.172981262207031, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4563196897506714, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08161462098360062, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.036333270370960236, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.010537147521973, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.4985666275024414, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.233051300048828, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02571420557796955, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15952084958553314, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15980708599090576, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011834162287414074, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19998301565647125, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006708953296765685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2760433852672577, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.027168894186615944, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7848911285400391, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16396.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23766368627548218, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4933218955993652, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2170395851135254, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0385618209838867, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15893395245075226, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.051533699035645, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005349674262106419, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5207262635231018, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.024229377508163452, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.2154812812805176, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20562.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7648458480834961, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.116179347038269, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.766985893249512, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.281106948852539, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.21109145879745483, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3098094165325165, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.579370498657227, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4961916506290436, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.429902076721191, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02518889307975769, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1588897705078125, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15932361781597137, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011474893428385258, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19967021048069, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00565362861379981, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5397637486457825, "metrics/particle/momentum/all/accepted/pz_bias": -0.044293180108070374, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7533241510391235, "metrics/particle/momentum/all/accepted/samples": 26001.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2699427306652069, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4398369789123535, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4092758893966675, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.2323524951934814, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16116131842136383, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.128107070922852, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01395164243876934, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9003718495368958, "metrics/particle/momentum/all/assigned/pz_bias": -0.04627630487084389, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3301544189453125, "metrics/particle/momentum/all/assigned/samples": 32756.0, "metrics/particle/momentum/all/efficiency": 0.7346160411834717, "metrics/particle/momentum/all/event/energy_bias": -7.890857696533203, "metrics/particle/momentum/all/event/p3_error_mean": 23.412960052490234, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.567700386047363, "metrics/particle/momentum/all/event/px_bias": -0.4760829210281372, "metrics/particle/momentum/all/event/py_bias": -0.5811751484870911, "metrics/particle/momentum/all/event/pz_abs_mean": 17.30573272705078, "metrics/particle/momentum/all/event/pz_bias": -5.896329402923584, "metrics/particle/momentum/all/event/pz_residual_rms": 27.5888729095459, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0272519588470459, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14999592304229736, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14950883388519287, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010485023260116577, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18843650817871094, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005750390235334635, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5698355436325073, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.06112492457032204, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6871341466903687, "metrics/particle/momentum/class_1/accepted/samples": 14086.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1407855898141861, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.42360132932662964, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4803889989852905, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.32502007484436035, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09382364153862, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.62857723236084, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009611806832253933, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7430911064147949, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.053229089826345444, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.384798288345337, "metrics/particle/momentum/class_1/assigned/samples": 16126.0, "metrics/particle/momentum/class_1/efficiency": 0.8441301584243774, "metrics/particle/momentum/class_1/event/energy_bias": 3.7026028633117676, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.49142837524414, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.409126281738281, "metrics/particle/momentum/class_1/event/px_bias": -0.2729355990886688, "metrics/particle/momentum/class_1/event/py_bias": -1.1788008213043213, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.160415649414062, "metrics/particle/momentum/class_1/event/pz_bias": -3.774048089981079, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.551252365112305, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0335383266210556, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2013845443725586, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2057860791683197, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.020507758483290672, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24494639039039612, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03019844740629196, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.826763391494751, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0022029662504792213, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.213599681854248, "metrics/particle/momentum/class_2/accepted/samples": 1159.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6619705557823181, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.935417115688324, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0791171789169312, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7046399712562561, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3231883943080902, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.909470558166504, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.052657004445791245, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7664365768432617, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11098727583885193, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.684662342071533, "metrics/particle/momentum/class_2/assigned/samples": 2070.0, "metrics/particle/momentum/class_2/efficiency": 0.3893181085586548, "metrics/particle/momentum/class_2/event/energy_bias": -9.470396995544434, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.85640525817871, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.722735404968262, "metrics/particle/momentum/class_2/event/px_bias": 0.4233410656452179, "metrics/particle/momentum/class_2/event/py_bias": -0.8242913484573364, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.042180061340332, "metrics/particle/momentum/class_2/event/pz_bias": -0.4853224754333496, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.3897647857666, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02146311104297638, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1653607189655304, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16681669652462006, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016005946323275566, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20809349417686462, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021233472507447004, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3199644684791565, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.032824963331222534, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2281845808029175, "metrics/particle/momentum/class_3/accepted/samples": 10361.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3619725704193115, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.6408097743988037, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.5139248371124268, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.3409087657928467, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2142907977104187, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.369140625, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012513331137597561, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7775921821594238, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05923029035329819, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4804203510284424, "metrics/particle/momentum/class_3/assigned/samples": 14065.0, "metrics/particle/momentum/class_3/efficiency": 0.6813309788703918, "metrics/particle/momentum/class_3/event/energy_bias": 0.8747691512107849, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.959200859069824, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.728994369506836, "metrics/particle/momentum/class_3/event/px_bias": -0.20054039359092712, "metrics/particle/momentum/class_3/event/py_bias": 1.045951247215271, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.398555755615234, "metrics/particle/momentum/class_3/event/pz_bias": -2.0900344848632812, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.59354305267334, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027986954897642136, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17564070224761963, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16994138062000275, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.039808206260204315, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2064146101474762, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01923076994717121, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1624666452407837, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3596719801425934, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.558826208114624, "metrics/particle/momentum/class_4/accepted/samples": 260.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2832367420196533, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.4186196327209473, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1699795722961426, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0340683460235596, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2069970816373825, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.37429141998291, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02332361601293087, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2779980897903442, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08001388609409332, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4110724925994873, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.7142857313156128, "metrics/particle/momentum/class_4/event/energy_bias": -1.7327383756637573, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.9765453338623047, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.055264472961426, "metrics/particle/momentum/class_4/event/px_bias": -0.15752257406711578, "metrics/particle/momentum/class_4/event/py_bias": 0.1712290644645691, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8007879257202148, "metrics/particle/momentum/class_4/event/pz_bias": 0.4439607560634613, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.36749792098999, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01880337856709957, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19315876066684723, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1889864206314087, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.012982144020497799, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2303779125213623, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029629629105329514, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0228350162506104, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3454059958457947, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.7785909175872803, "metrics/particle/momentum/class_5/accepted/samples": 135.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08791087567806244, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5290499329566956, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5201707482337952, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2926788330078125, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07894736528396606, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.6080095767974854, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05921052768826485, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.68278169631958, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5366372466087341, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.5527448654174805, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.849056601524353, "metrics/particle/momentum/class_5/event/energy_bias": -1.2650947570800781, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.8275532722473145, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.11393666267395, "metrics/particle/momentum/class_5/event/px_bias": -0.2684253454208374, "metrics/particle/momentum/class_5/event/py_bias": 0.20473672449588776, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5173852443695068, "metrics/particle/momentum/class_5/event/pz_bias": 0.009115006774663925, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.804782390594482, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42860013246536255, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4303537607192993, "metrics/set_slot/angular_recall_all_dr0p10": 0.9387766718864441, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7893147468566895, "metrics/set_slot/soft_count_bias_mean": -1.5620341300964355}, "step_train_time": 4.5420098304748535, "step_valid_time": 84218.33002114296, "step_total_time": 84222.87203145027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031910881585384, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1859553455782409, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2991, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2825, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2794, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9332672613148332, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9445001671681712, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388501163177135, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05549983283182882, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9230260984473075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9341357405549984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9285476902625457, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9332672613148332, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9901189666269611, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1707968657769332, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2113, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2106, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9341290893015031, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9471089197669207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9405742265746717, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052891080233079334, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9310344827586207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9439713133124159, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9374582684175384, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9341290893015031, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994539604911821, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16833703288641677, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1165, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9114906832298136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9310071371927042, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9211455472734406, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0689928628072958, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9045031055900621, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9238699444885012, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9140839544919577, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9114906832298136} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_34000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..4df72182b9bcc5a0b7d444d32acfd62c7ddb58e7 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.6107636914913777, "train": {"Classification_binary": 0.16810330748558044, "Cardinality": 0.13189305365085602, "Classification": 0.08251319825649261, "Regression_pt": 0.16835154592990875, "Regression_eta": 0.0017801995854824781, "Regression_sin_phi": 0.0010403539054095745, "Regression_cos_phi": 0.0009754070779308677, "Regression_energy": 0.1663045883178711, "HitGrouping": 0.22226357460021973, "Auxiliary": 0.24792207777500153, "Total": 1.1988178491592407, "Total_unweighted": 0.5890686005586758}, "train_group": {}, "valid": {"Classification_binary": 0.19133392969767252, "Cardinality": 0.13148597876230875, "Classification": 0.09813348452250163, "Regression_pt": 0.16860204935073853, "Regression_eta": 0.0018287644100685914, "Regression_sin_phi": 0.0010325203960140545, "Regression_cos_phi": 0.001031905955945452, "Regression_energy": 0.16751362880071005, "HitGrouping": 0.22036872307459512, "Auxiliary": 0.26311131318410236, "Total": 1.2502777576446533, "Total_unweighted": 0.6294762831336508, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7591328024864197, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3889407217502594, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42306602001190186, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3830375075340271, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.38271644711494446, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5492585301399231, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7023583054542542, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3540829122066498, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42494070529937744, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.34753385186195374, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.36242198944091797, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6245340704917908, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7763145565986633, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3824786841869354, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40486520528793335, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.37685030698776245, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3647703528404236, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5220018625259399, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7641521096229553, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3820110857486725, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4125227928161621, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3762117326259613, "metrics/hit_grouping/calorimeter/all/object_purity": 0.370334655046463, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5399696230888367, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02594657614827156, "metrics/hit_grouping/calorimeter/background_recall": 0.6325130462646484, "metrics/hit_grouping/calorimeter/completeness": 0.44036340713500977, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4616314768791199, "metrics/hit_grouping/calorimeter/energy_purity": 0.45629605650901794, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4773131310939789, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.784171462059021, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3984218239784241, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34392356872558594, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3946954607963562, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3898393511772156, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4882054328918457, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7616549730300903, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3992973864078522, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4525715708732605, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.39135098457336426, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3875744342803955, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5557981729507446, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7243797183036804, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.32926124334335327, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5159903764724731, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.321311354637146, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3110840618610382, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.633042573928833, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7510824799537659, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.409911572933197, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49361568689346313, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.404255747795105, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.39842069149017334, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5755216479301453, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7387542724609375, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4907122254371643, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6185243129730225, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47997206449508667, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.47534361481666565, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5907901525497437, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7814527153968811, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.338106632232666, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2989822030067444, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.33278515934944153, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3267030715942383, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4952457547187805, "metrics/hit_grouping/calorimeter/purity": 0.4339476227760315, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8426172137260437, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07053407281637192, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15076923370361328, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07248840481042862, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03227296844124794, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0630769208073616, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7773486971855164, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14623548090457916, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11630558967590332, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14435099065303802, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13531000912189484, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38084378838539124, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7617206573486328, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.40221086144447327, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4351390302181244, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3960210382938385, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3910578191280365, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5592864751815796, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7248409390449524, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4246861934661865, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.44829896092414856, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.46035709977149963, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6125224232673645, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7832759022712708, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35501962900161743, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4790937602519989, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46498754620552063, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.454290509223938, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6704950332641602, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43529966473579407, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3908885419368744, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4026191830635071, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6777979135513306, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6971704959869385, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4251842200756073, "metrics/hit_grouping/tracker/all/object_completeness": 0.41646289825439453, "metrics/hit_grouping/tracker/all/object_purity": 0.42612123489379883, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.638343095779419, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010632684454321861, "metrics/hit_grouping/tracker/background_recall": 0.050701890140771866, "metrics/hit_grouping/tracker/completeness": 0.4310263991355896, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5111151933670044, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7091964483261108, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40847378969192505, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46739330887794495, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4821464717388153, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6269704103469849, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6879683136940002, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45231297612190247, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3994358479976654, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4086056649684906, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6550542712211609, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6802515983581543, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.433454692363739, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32149559259414673, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3203631043434143, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6458333134651184, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7214702367782593, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44536304473876953, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43983420729637146, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.44219785928726196, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6044212579727173, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7336944341659546, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4513981342315674, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45273274183273315, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4484652280807495, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5778961181640625, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6590919494628906, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3942870497703552, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37963855266571045, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40124860405921936, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6931632161140442, "metrics/hit_grouping/tracker/purity": 0.41322436928749084, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8815154433250427, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02128583751618862, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.037076886743307114, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03588385134935379, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0873153805732727, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6706916689872742, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.228615865111351, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.235892653465271, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.23354816436767578, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5728356838226318, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6751360893249512, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5024496912956238, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48857593536376953, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5010754466056824, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7206776738166809, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4192748963832855, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5386533141136169, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5906550288200378, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5903122425079346, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06645245105028152, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05950741842389107, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.135147094726562, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28719139099121094, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1567515730857849, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8571824431419373, "metrics/particle/class_1/pid_accuracy": 0.9630210995674133, "metrics/particle/class_2/efficiency": 0.411824107170105, "metrics/particle/class_2/pid_accuracy": 0.6319803595542908, "metrics/particle/class_3/efficiency": 0.692746639251709, "metrics/particle/class_3/pid_accuracy": 0.9568300843238831, "metrics/particle/class_4/efficiency": 0.699711263179779, "metrics/particle/class_4/pid_accuracy": 0.36038514971733093, "metrics/particle/class_5/efficiency": 0.8840000033378601, "metrics/particle/class_5/pid_accuracy": 0.40045249462127686, "metrics/particle/count/bias_mean": -3.5227863788604736, "metrics/particle/count/mae": 4.818359375, "metrics/particle/count/prediction_mean": 66.533203125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07808449119329453, "metrics/particle/event/energy_response_mean": 0.9611058831214905, "metrics/particle/event/met_abs_error": 8.09178352355957, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0653165951371193, "metrics/particle/event/scalar_pt_response_mean": 0.9692366123199463, "metrics/particle/event/vector_pt_closure": 0.06286658346652985, "metrics/particle/matched/delta_eta_abs_mean": 0.01563182845711708, "metrics/particle/matched/delta_phi_abs_mean": 0.015461798757314682, "metrics/particle/matched/delta_r_mean": 0.024653730913996696, "metrics/particle/matched/energy_relative_abs_mean": 0.15828277170658112, "metrics/particle/matched/pid_accuracy": 0.9368948340415955, "metrics/particle/matched/pt_relative_abs_mean": 0.15717655420303345, "metrics/particle/matching/duplicate_fraction": 0.06284064799547195, "metrics/particle/matching/efficiency": 0.7476627826690674, "metrics/particle/matching/f1": 0.7669458389282227, "metrics/particle/matching/purity": 0.7872498631477356, "metrics/particle/matching/target_coverage_dr0p05": 0.6929910778999329, "metrics/particle/matching/target_coverage_dr0p10": 0.8043975234031677, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5339200496673584, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7904670834541321, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023606402799487114, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15430359542369843, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15458467602729797, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0006961789331398904, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19433943927288055, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009861440397799015, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.823754608631134, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07026635855436325, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2138094902038574, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8011.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2917068600654602, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.494520664215088, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4443223476409912, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2599859237670898, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.156206414103508, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.733169555664062, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02022315189242363, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2392290830612183, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12191519886255264, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4743247032165527, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10038.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7325347661972046, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.027611255645752, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.228961944580078, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.45674991607666, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2794257700443268, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.48283615708351135, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.858650207519531, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.6650646924972534, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.857535362243652, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023706665262579918, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16629855334758759, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16510999202728271, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016524646431207657, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20485398173332214, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027839643880724907, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6392734050750732, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07371709495782852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9239301681518555, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1796.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4970572888851166, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1139254570007324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.514622211456299, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.3294293880462646, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20000000298023224, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.093339920043945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.059470467269420624, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.655843496322632, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09070449322462082, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.367150783538818, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2455.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5945051312446594, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.039216041564941, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.748634338378906, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.538325309753418, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.011772334575653076, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11083745211362839, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.53529167175293, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.40399524569511414, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.151214599609375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024686578661203384, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15719102323055267, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1556159406900406, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.005950832739472389, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1956917941570282, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009072763496078551, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27073749899864197, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.029339589178562164, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8303055167198181, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16533.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2518530786037445, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.363006591796875, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0624656677246094, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8739929795265198, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1579277366399765, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.100314140319824, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005884906742721796, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5084095001220703, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.040379833430051804, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1548831462860107, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20731.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7712366580963135, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.32147216796875, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.678108215332031, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.948820114135742, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.42153167724609375, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8021326065063477, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.891678333282471, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0051246881484985, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.957650184631348, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02429123967885971, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15693385899066925, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1559496521949768, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004650200717151165, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1959211379289627, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00546697061508894, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5322450995445251, "metrics/particle/momentum/all/accepted/pz_bias": 0.0448128767311573, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7242841720581055, "metrics/particle/momentum/all/accepted/samples": 26340.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2820128798484802, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.384335994720459, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.285139799118042, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0981589555740356, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16051650047302246, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.785804748535156, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014176499098539352, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8878917098045349, "metrics/particle/momentum/all/assigned/pz_bias": 0.06873280555009842, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2488389015197754, "metrics/particle/momentum/all/assigned/samples": 33224.0, "metrics/particle/momentum/all/efficiency": 0.7441939115524292, "metrics/particle/momentum/all/event/energy_bias": -11.388298988342285, "metrics/particle/momentum/all/event/p3_error_mean": 22.380598068237305, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.173235893249512, "metrics/particle/momentum/all/event/px_bias": 0.7127299308776855, "metrics/particle/momentum/all/event/py_bias": -2.395806312561035, "metrics/particle/momentum/all/event/pz_abs_mean": 16.646392822265625, "metrics/particle/momentum/all/event/pz_bias": 1.2661941051483154, "metrics/particle/momentum/all/event/pz_residual_rms": 25.786134719848633, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025758802890777588, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14694136381149292, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14717675745487213, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0032350043766200542, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18564854562282562, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005590105429291725, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5672740340232849, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06417547166347504, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7648138999938965, "metrics/particle/momentum/class_1/accepted/samples": 14311.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1382940411567688, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.38003647327423096, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.43381044268608093, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.268696129322052, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08541692048311234, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.1106719970703125, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009114422835409641, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7120304703712463, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.09396005421876907, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2184650897979736, "metrics/particle/momentum/class_1/assigned/samples": 16238.0, "metrics/particle/momentum/class_1/efficiency": 0.8576136827468872, "metrics/particle/momentum/class_1/event/energy_bias": 2.7987518310546875, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.623451232910156, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.621051788330078, "metrics/particle/momentum/class_1/event/px_bias": -1.9750136137008667, "metrics/particle/momentum/class_1/event/py_bias": -0.6844441890716553, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.80340576171875, "metrics/particle/momentum/class_1/event/pz_bias": 2.9253313541412354, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.71050453186035, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03302847966551781, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20130476355552673, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2006036639213562, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02939539961516857, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24127306044101715, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025326797738671303, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7647639513015747, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13424724340438843, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.994946241378784, "metrics/particle/momentum/class_2/accepted/samples": 1224.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6804978847503662, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7832958102226257, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0552886724472046, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6571977734565735, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3333333432674408, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.5659403800964355, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.054644808173179626, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7042624950408936, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07372421771287918, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.445188045501709, "metrics/particle/momentum/class_2/assigned/samples": 2196.0, "metrics/particle/momentum/class_2/efficiency": 0.4111521542072296, "metrics/particle/momentum/class_2/event/energy_bias": -11.803186416625977, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.954721450805664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.723426818847656, "metrics/particle/momentum/class_2/event/px_bias": 1.8644254207611084, "metrics/particle/momentum/class_2/event/py_bias": -1.5730557441711426, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.888375282287598, "metrics/particle/momentum/class_2/event/pz_bias": -2.0638909339904785, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.43320655822754, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021303804591298103, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16512587666511536, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1625255048274994, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010450315661728382, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20304550230503082, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022096263710409403, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.30837807059288025, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00788166094571352, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0097376108169556, "metrics/particle/momentum/class_3/accepted/samples": 10409.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.38484954833984375, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5446741580963135, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2252933979034424, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.045053720474243, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.218754380941391, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.66739845275879, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012526242062449455, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.778544545173645, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.037462055683135986, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4538040161132812, "metrics/particle/momentum/class_3/assigned/samples": 14290.0, "metrics/particle/momentum/class_3/efficiency": 0.6844874024391174, "metrics/particle/momentum/class_3/event/energy_bias": 0.34542256593704224, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.840021133422852, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.030788421630859, "metrics/particle/momentum/class_3/event/px_bias": 1.1976988315582275, "metrics/particle/momentum/class_3/event/py_bias": -0.19269369542598724, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.205836296081543, "metrics/particle/momentum/class_3/event/pz_bias": -0.22200115025043488, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.664981842041016, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025552839040756226, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16361425817012787, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1599626988172531, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.027495676651597023, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20716001093387604, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01550387591123581, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.003507375717163, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2546691298484802, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1027963161468506, "metrics/particle/momentum/class_4/accepted/samples": 258.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3385710120201111, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.7833781242370605, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.208025932312012, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.066098213195801, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21098266541957855, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.92630386352539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03179190680384636, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3663132190704346, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2191019207239151, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.7953810691833496, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.708791196346283, "metrics/particle/momentum/class_4/event/energy_bias": -1.1614644527435303, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0229666233062744, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0634615421295166, "metrics/particle/momentum/class_4/event/px_bias": 0.08938812464475632, "metrics/particle/momentum/class_4/event/py_bias": -0.08786743879318237, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8810415267944336, "metrics/particle/momentum/class_4/event/pz_bias": 0.5522102117538452, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.504040718078613, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01758178509771824, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16923652589321136, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16615842282772064, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.02681952714920044, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21481341123580933, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9724185466766357, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3630976378917694, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.21545934677124, "metrics/particle/momentum/class_5/accepted/samples": 138.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08414134383201599, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5427446961402893, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.52229905128479, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.31333887577056885, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09740259498357773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.4938552379608154, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08441558480262756, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.6016695499420166, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.09860625863075256, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.15480375289917, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.8679245114326477, "metrics/particle/momentum/class_5/event/energy_bias": -1.5678226947784424, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.735222816467285, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0569751262664795, "metrics/particle/momentum/class_5/event/px_bias": -0.4637690484523773, "metrics/particle/momentum/class_5/event/py_bias": 0.1422547847032547, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4270529747009277, "metrics/particle/momentum/class_5/event/pz_bias": 0.07454423606395721, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.566468238830566, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4097486436367035, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4111489951610565, "metrics/set_slot/angular_recall_all_dr0p10": 0.9402356743812561, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7937382459640503, "metrics/set_slot/soft_count_bias_mean": -0.31729644536972046}, "step_train_time": 5.70300555229187, "step_valid_time": 86895.76424479485, "step_total_time": 86901.46725130081, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9779186584505296, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15812297066714498, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2858, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2830, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9441691443673604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9463576158940398, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9452621134446834, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05364238410596026, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9349190617773373, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370860927152318, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9360013229700678, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9441691443673604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9702053591136309, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16080521396447733, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2105, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2093, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9305923961096375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9426780116435289, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9365962180200222, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057321988356471114, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9252873563218391, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9373040752351097, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9312569521690768, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9305923961096375, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9715961227660689, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.157158348609981, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1264, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1171, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122670807453416, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9295886075949367, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.920846394984326, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07041139240506329, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9091614906832298, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9264240506329114, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9177115987460815, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122670807453416} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_35000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..eb66305ca0a798b91acbb94c47855651b11c369a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6581390941823007, "train": {"Classification_binary": 0.1882084310054779, "Cardinality": 0.15186303853988647, "Classification": 0.09230394661426544, "Regression_pt": 0.1738176792860031, "Regression_eta": 0.0017365706153213978, "Regression_sin_phi": 0.0009536322322674096, "Regression_cos_phi": 0.0010859329486265779, "Regression_energy": 0.17216290533542633, "HitGrouping": 0.2292889654636383, "Auxiliary": 0.27112913131713867, "Total": 1.2808629274368286, "Total_unweighted": 0.6302690980373882}, "train_group": {}, "valid": {"Classification_binary": 0.19274147351582846, "Cardinality": 0.1359423597653707, "Classification": 0.09253284335136414, "Regression_pt": 0.17541265487670898, "Regression_eta": 0.0017601513924698036, "Regression_sin_phi": 0.0010037078366925318, "Regression_cos_phi": 0.0010337350734819968, "Regression_energy": 0.17449259757995605, "HitGrouping": 0.2268505096435547, "Auxiliary": 0.26942994197209674, "Total": 1.2680133978525798, "Total_unweighted": 0.6389771636265019, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7547107338905334, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.36989352107048035, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41683197021484375, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3624141812324524, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36430788040161133, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5515254735946655, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6944103837013245, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3318001925945282, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41850221157073975, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3240921199321747, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34932953119277954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.635377824306488, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7743363380432129, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3678625524044037, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.392779678106308, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3614518940448761, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3504786491394043, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5083152651786804, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7608519792556763, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.36528146266937256, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4028267562389374, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.358410507440567, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3548257350921631, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5336170196533203, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.027067013084888458, "metrics/hit_grouping/calorimeter/background_recall": 0.6411373019218445, "metrics/hit_grouping/calorimeter/completeness": 0.42103517055511475, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44308555126190186, "metrics/hit_grouping/calorimeter/energy_purity": 0.4381878674030304, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.488104909658432, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7790652513504028, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3831183910369873, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33645865321159363, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37865978479385376, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37641626596450806, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4816960394382477, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7580541372299194, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3782750368118286, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4422610402107239, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36972343921661377, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.36744192242622375, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5524017214775085, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7252159118652344, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.31398576498031616, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5022121071815491, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3037603199481964, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2959783375263214, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6239413619041443, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7464298605918884, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.38685932755470276, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48284024000167847, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3792259097099304, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3773922324180603, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5806291103363037, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7517135143280029, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4792254865169525, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6420722007751465, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.46947959065437317, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.45681101083755493, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5620093941688538, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7775259613990784, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3275032341480255, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.28619256615638733, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.32181990146636963, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3175150454044342, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.47944289445877075, "metrics/hit_grouping/calorimeter/purity": 0.4153788089752197, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8235543370246887, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0777052566409111, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13692307472229004, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07905860245227814, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03484255447983742, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7453122138977051, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14769086241722107, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10319270193576813, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14639726281166077, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13863100111484528, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.33295324444770813, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7604092955589294, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38392582535743713, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4257279634475708, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3765588104724884, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3740805387496948, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5556138753890991, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7274473905563354, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4278242588043213, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.43351107835769653, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4329484701156616, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.596682608127594, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7757784724235535, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.33875492215156555, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4648655354976654, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4605214595794678, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4632641673088074, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6826571822166443, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4287627041339874, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38255488872528076, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3790838122367859, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6665353775024414, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7046566605567932, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4208860695362091, "metrics/hit_grouping/tracker/all/object_completeness": 0.40559279918670654, "metrics/hit_grouping/tracker/all/object_purity": 0.40296754240989685, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6273379921913147, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00811329297721386, "metrics/hit_grouping/tracker/background_recall": 0.040987975895404816, "metrics/hit_grouping/tracker/completeness": 0.42459818720817566, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5218001008033752, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.709199070930481, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41175398230552673, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45923253893852234, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4643222689628601, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6263325810432434, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6978510618209839, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43917760252952576, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38752004504203796, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.38564541935920715, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6350656747817993, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.7018038630485535, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42172330617904663, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3057222366333008, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.28517311811447144, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6213592290878296, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7331978678703308, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4399712383747101, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42446252703666687, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.41408491134643555, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5859094262123108, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7632489204406738, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46737682819366455, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4635150730609894, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42237013578414917, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5346205234527588, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6571719646453857, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38784828782081604, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37082555890083313, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3850727081298828, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6976118087768555, "metrics/hit_grouping/tracker/purity": 0.40602773427963257, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.877822995185852, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014769765548408031, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03171150013804436, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03372332081198692, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07384882867336273, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6757994890213013, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20165888965129852, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1922866404056549, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.19095060229301453, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5800933241844177, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6844120025634766, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5010330080986023, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4806853234767914, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4772831201553345, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7079275250434875, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.43436360359191895, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5627768039703369, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6251831650733948, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.606670081615448, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05136372521519661, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07586529105901718, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.8709659576416, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28459253907203674, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15415272116661072, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8524535298347473, "metrics/particle/class_1/pid_accuracy": 0.9611215591430664, "metrics/particle/class_2/efficiency": 0.4040834605693817, "metrics/particle/class_2/pid_accuracy": 0.6068850755691528, "metrics/particle/class_3/efficiency": 0.6857529282569885, "metrics/particle/class_3/pid_accuracy": 0.9549819827079773, "metrics/particle/class_4/efficiency": 0.6920115351676941, "metrics/particle/class_4/pid_accuracy": 0.3602225184440613, "metrics/particle/class_5/efficiency": 0.871999979019165, "metrics/particle/class_5/pid_accuracy": 0.37844038009643555, "metrics/particle/count/bias_mean": -2.8450520038604736, "metrics/particle/count/mae": 4.467447757720947, "metrics/particle/count/prediction_mean": 67.2109375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07174527645111084, "metrics/particle/event/energy_response_mean": 0.9862892031669617, "metrics/particle/event/met_abs_error": 8.94836711883545, "metrics/particle/event/scalar_pt_relative_abs_error": 0.060682158917188644, "metrics/particle/event/scalar_pt_response_mean": 1.0010641813278198, "metrics/particle/event/vector_pt_closure": 0.06833409518003464, "metrics/particle/matched/delta_eta_abs_mean": 0.015238071791827679, "metrics/particle/matched/delta_phi_abs_mean": 0.015570799820125103, "metrics/particle/matched/delta_r_mean": 0.024357706308364868, "metrics/particle/matched/energy_relative_abs_mean": 0.16200052201747894, "metrics/particle/matched/pid_accuracy": 0.9340767860412598, "metrics/particle/matched/pt_relative_abs_mean": 0.162117138504982, "metrics/particle/matching/duplicate_fraction": 0.07023712992668152, "metrics/particle/matching/efficiency": 0.7416408061981201, "metrics/particle/matching/f1": 0.7570123672485352, "metrics/particle/matching/purity": 0.7730345726013184, "metrics/particle/matching/target_coverage_dr0p05": 0.6958719491958618, "metrics/particle/matching/target_coverage_dr0p10": 0.8090348243713379, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5177127718925476, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7851142287254333, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02285184897482395, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1580003947019577, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15897831320762634, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020324109122157097, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19957132637500763, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009904761798679829, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8418012857437134, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05473765358328819, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2319436073303223, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7875.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3005806505680084, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2965731620788574, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3148458003997803, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1478651762008667, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1721958965063095, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.425618171691895, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021228278055787086, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.275451421737671, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.1196722462773323, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.457427740097046, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10128.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7200987339019775, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7644014358520508, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.229988098144531, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.734460830688477, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.45863986015319824, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.9410703182220459, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.685017585754395, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.3817449808120728, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.29731559753418, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02442340739071369, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1722116768360138, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17050589621067047, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028787869960069656, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2120337039232254, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03503893315792084, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.792847990989685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05781403183937073, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.7983317375183105, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1798.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.47039297223091125, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.2793097496032715, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.1167731285095215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.9465253353118896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22311720252037048, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 97.05267333984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07732581347227097, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0955255031585693, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03846392408013344, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.211087226867676, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2483.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5951671600341797, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.0050156116485596, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.886285781860352, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5297359228134155, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.1797856092453003, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.02792707271873951, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.694259643554688, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.10785052180290222, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.811153411865234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024618009105324745, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16172444820404053, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16166867315769196, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017202451825141907, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20242683589458466, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011503995629027486, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2827915847301483, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.021468011662364006, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8465373516082764, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16516.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2366427481174469, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3020566701889038, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0606924295425415, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8821966648101807, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16493606567382812, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.975866794586182, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005143736023455858, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5031798481941223, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.027601653710007668, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8826625347137451, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20802.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7704436182975769, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.1539877951145172, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.869941711425781, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.207820892333984, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.889993667602539, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.636019706726074, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.8419599533081055, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6930086612701416, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.76688814163208, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024073565378785133, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1613246500492096, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16146640479564667, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0189365241676569, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20224936306476593, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006109435111284256, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5545576810836792, "metrics/particle/momentum/all/accepted/pz_bias": 0.033967483788728714, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.878988265991211, "metrics/particle/momentum/all/accepted/samples": 26189.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2733938694000244, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.373016595840454, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3648349046707153, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1904425621032715, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17146021127700806, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 27.68571662902832, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01538323424756527, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9299107789993286, "metrics/particle/momentum/all/assigned/pz_bias": 0.056316886097192764, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.483534336090088, "metrics/particle/momentum/all/assigned/samples": 33413.0, "metrics/particle/momentum/all/efficiency": 0.7399276494979858, "metrics/particle/momentum/all/event/energy_bias": -3.615429401397705, "metrics/particle/momentum/all/event/p3_error_mean": 22.062089920043945, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.677651405334473, "metrics/particle/momentum/all/event/px_bias": 2.528419017791748, "metrics/particle/momentum/all/event/py_bias": 3.6050174236297607, "metrics/particle/momentum/all/event/pz_abs_mean": 15.731555938720703, "metrics/particle/momentum/all/event/pz_bias": 2.1826043128967285, "metrics/particle/momentum/all/event/pz_residual_rms": 25.5886287689209, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025412768125534058, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15249335765838623, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1534939855337143, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012146532535552979, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19322678446769714, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005883588921278715, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5762732028961182, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.02742028795182705, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7975666522979736, "metrics/particle/momentum/class_1/accepted/samples": 14277.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12771108746528625, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3647642433643341, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4113966226577759, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25410428643226624, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09440088272094727, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.391294479370117, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010904190130531788, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7703567147254944, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0524580143392086, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8669698238372803, "metrics/particle/momentum/class_1/assigned/samples": 16324.0, "metrics/particle/momentum/class_1/efficiency": 0.85557621717453, "metrics/particle/momentum/class_1/event/energy_bias": 7.149785995483398, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.396108627319336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.148569107055664, "metrics/particle/momentum/class_1/event/px_bias": 0.10146920382976532, "metrics/particle/momentum/class_1/event/py_bias": 1.2491199970245361, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.15177059173584, "metrics/particle/momentum/class_1/event/pz_bias": 0.5216577053070068, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.698959350585938, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033427659422159195, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2094990462064743, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21129992604255676, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022148407995700836, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25016477704048157, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.033927056938409805, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9391621351242065, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.2826642096042633, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.328560829162598, "metrics/particle/momentum/class_2/accepted/samples": 1179.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7401316165924072, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8596127033233643, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1666646003723145, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7442327737808228, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3626966178417206, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.716373920440674, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0660674124956131, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9296414852142334, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.473664253950119, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.9555511474609375, "metrics/particle/momentum/class_2/assigned/samples": 2225.0, "metrics/particle/momentum/class_2/efficiency": 0.3960362672805786, "metrics/particle/momentum/class_2/event/energy_bias": -11.04740047454834, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.985815048217773, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.353775024414062, "metrics/particle/momentum/class_2/event/px_bias": 1.7649160623550415, "metrics/particle/momentum/class_2/event/py_bias": 1.7995269298553467, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.537860870361328, "metrics/particle/momentum/class_2/event/pz_bias": 1.5597937107086182, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.7491512298584, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02116822823882103, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16742198169231415, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1663103997707367, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03284642845392227, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2078203558921814, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0026114711072295904, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.33512458205223083, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0037076156586408615, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4167366027832031, "metrics/particle/momentum/class_3/accepted/samples": 10339.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3666835129261017, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.596385955810547, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.479250192642212, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.323632001876831, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22896936535835266, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 41.99623107910156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011908077634871006, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7761658430099487, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0077890390530228615, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.2891812324523926, "metrics/particle/momentum/class_3/assigned/samples": 14360.0, "metrics/particle/momentum/class_3/efficiency": 0.6798842549324036, "metrics/particle/momentum/class_3/event/energy_bias": 3.8584680557250977, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.112963676452637, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.591461181640625, "metrics/particle/momentum/class_3/event/px_bias": 0.6333864331245422, "metrics/particle/momentum/class_3/event/py_bias": 0.1612119972705841, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.71806812286377, "metrics/particle/momentum/class_3/event/pz_bias": -0.5224884748458862, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.505858421325684, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026281140744686127, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17526230216026306, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.174136683344841, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05043661221861839, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21010136604309082, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.019455252215266228, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1744569540023804, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.657090425491333, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6819326877593994, "metrics/particle/momentum/class_4/accepted/samples": 257.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3476022481918335, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8968969583511353, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7964797019958496, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6720538139343262, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22507122159004211, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.172904014587402, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022792022675275803, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3058083057403564, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.44623833894729614, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.643367052078247, "metrics/particle/momentum/class_4/assigned/samples": 351.0, "metrics/particle/momentum/class_4/efficiency": 0.7060439586639404, "metrics/particle/momentum/class_4/event/energy_bias": -1.4045119285583496, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0439858436584473, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.085808038711548, "metrics/particle/momentum/class_4/event/px_bias": 0.11915562301874161, "metrics/particle/momentum/class_4/event/py_bias": 0.032720617949962616, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8204131126403809, "metrics/particle/momentum/class_4/event/pz_bias": 0.7223917841911316, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.634912014007568, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.019129369407892227, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18077054619789124, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17409509420394897, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.028729017823934555, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21845541894435883, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029197080060839653, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7729822397232056, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.30928125977516174, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.459517478942871, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10311377048492432, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.38981738686561584, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.38673916459083557, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11827369034290314, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09150326997041702, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.047680616378784, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06535948067903519, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.439699411392212, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4790228605270386, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.87155294418335, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.8616352081298828, "metrics/particle/momentum/class_5/event/energy_bias": -2.1717703342437744, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7906384468078613, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0826194286346436, "metrics/particle/momentum/class_5/event/px_bias": -0.09050799906253815, "metrics/particle/momentum/class_5/event/py_bias": 0.3624378442764282, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4942431449890137, "metrics/particle/momentum/class_5/event/pz_bias": -0.09875032305717468, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.646117687225342, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41738399863243103, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4195467531681061, "metrics/set_slot/angular_recall_all_dr0p10": 0.9405888319015503, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7993885278701782, "metrics/set_slot/soft_count_bias_mean": 0.6307179927825928}, "step_train_time": 5.192210674285889, "step_valid_time": 89498.73494386673, "step_total_time": 89503.92715477943, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001516641363407, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1731069626282542, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2994, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2822, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2797, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9322761810373307, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9425517702070808, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9373858163095832, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05744822979291917, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.92401717872481, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9342017368069472, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9290815479156286, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9322761810373307, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999143117468368, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16276729915970134, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2126, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9398762157382847, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9465716829919858, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9432120674356699, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053428317008014245, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9350132625994695, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9416740872662511, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9383318544809228, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9398762157382847, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0123066136131105, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.160706152609146, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9184782608695652, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9256651017214398, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9220576773187841, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07433489827856025, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9130434782608695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.92018779342723, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.916601714731099, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9184782608695652} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_36000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..502b4f699fe3f49341e5fed68f51c5ce91a627f5 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.7055144968732234, "train": {"Classification_binary": 0.1896074116230011, "Cardinality": 0.14592058956623077, "Classification": 0.09758386760950089, "Regression_pt": 0.1584894061088562, "Regression_eta": 0.0016912692226469517, "Regression_sin_phi": 0.0008985053282231092, "Regression_cos_phi": 0.0008265228243544698, "Regression_energy": 0.1573246717453003, "HitGrouping": 0.2213078737258911, "Auxiliary": 0.26081520318984985, "Total": 1.2206008434295654, "Total_unweighted": 0.606421654461883}, "train_group": {}, "valid": {"Classification_binary": 0.1893681287765503, "Cardinality": 0.1307233770688375, "Classification": 0.09350850184758504, "Regression_pt": 0.1632958948612213, "Regression_eta": 0.001789191582550605, "Regression_sin_phi": 0.0009468585097541412, "Regression_cos_phi": 0.0009813754198451836, "Regression_energy": 0.16251549124717712, "HitGrouping": 0.22160770495732626, "Auxiliary": 0.2605076829592387, "Total": 1.2266050974527996, "Total_unweighted": 0.6124054422446837, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7552082538604736, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.37614133954048157, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4190988838672638, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3686651587486267, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.370195209980011, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5709832906723022, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6903855204582214, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.32904812693595886, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4232463538646698, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.32066696882247925, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34755006432533264, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6519823670387268, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7810416221618652, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.37382185459136963, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3852907717227936, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3674148917198181, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3577674329280853, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5161140561103821, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7645971179008484, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3705522119998932, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3995744585990906, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3636241555213928, "metrics/hit_grouping/calorimeter/all/object_purity": 0.36085015535354614, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5459574460983276, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02565266378223896, "metrics/hit_grouping/calorimeter/background_recall": 0.6302552223205566, "metrics/hit_grouping/calorimeter/completeness": 0.432171493768692, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4553311765193939, "metrics/hit_grouping/calorimeter/energy_purity": 0.44997069239616394, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.47943198680877686, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7812319993972778, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.38576966524124146, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33263659477233887, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.38061389327049255, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3805316984653473, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5047476887702942, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7616299986839294, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3857172429561615, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4369238317012787, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3771253526210785, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3789850175380707, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5585880875587463, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7324786186218262, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.32253849506378174, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5023385286331177, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.313592791557312, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3002924621105194, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6200227737426758, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7543443441390991, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.403708815574646, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48595452308654785, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3964693248271942, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3896964192390442, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5701027512550354, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7602136135101318, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.48884475231170654, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6164311766624451, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47866174578666687, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.46483632922172546, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5609628558158875, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7761805057525635, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3197696805000305, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2789607644081116, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3135928809642792, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3165321946144104, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5180795788764954, "metrics/hit_grouping/calorimeter/purity": 0.4257465898990631, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8499639630317688, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.061247747391462326, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12923076748847961, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.061652012169361115, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03426642715930939, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7758256793022156, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14685150980949402, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1077537089586258, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1443578153848648, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.136501744389534, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4053591787815094, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7621317505836487, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.39001113176345825, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.42212092876434326, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.38267168402671814, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.38071462512016296, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5646970272064209, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7317891716957092, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4339509904384613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4449829161167145, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.44212406873703003, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5980274677276611, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7890448570251465, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34324172139167786, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4709877073764801, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45812085270881653, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4593381881713867, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6724382042884827, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43797746300697327, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3905206322669983, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.39724379777908325, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.682759702205658, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7010177969932556, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4287266135215759, "metrics/hit_grouping/tracker/all/object_completeness": 0.41451147198677063, "metrics/hit_grouping/tracker/all/object_purity": 0.4165562093257904, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6371622681617737, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005368194077163935, "metrics/hit_grouping/tracker/background_recall": 0.026594799011945724, "metrics/hit_grouping/tracker/completeness": 0.4327821731567383, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5205762982368469, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.713748574256897, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4174942970275879, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4692092835903168, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47437745332717896, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.626423716545105, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.691905677318573, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45059967041015625, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.39342424273490906, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.39657601714134216, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6527698636054993, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6824387311935425, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43042072653770447, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3154950737953186, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3094300627708435, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6444174647331238, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7271436452865601, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45201292634010315, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43813633918762207, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.43270644545555115, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6003774404525757, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7415717244148254, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46671104431152344, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45463627576828003, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4395103454589844, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5739014744758606, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6598516702651978, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3917115330696106, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37667903304100037, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39148059487342834, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6962069869041443, "metrics/hit_grouping/tracker/purity": 0.41271156072616577, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8901754021644592, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.022589052096009254, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03371628001332283, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.032442737370729446, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07819287478923798, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6743051409721375, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2203214168548584, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21529172360897064, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2199517786502838, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5728356838226318, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6783559918403625, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5076441764831543, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4889395534992218, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.49113738536834717, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7204415202140808, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4054403305053711, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5608330965042114, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6195691823959351, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6005097031593323, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08028701692819595, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06970494985580444, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.21515464782715, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28894323110580444, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15850341320037842, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8622477054595947, "metrics/particle/class_1/pid_accuracy": 0.9658543467521667, "metrics/particle/class_2/efficiency": 0.40329816937446594, "metrics/particle/class_2/pid_accuracy": 0.6094575524330139, "metrics/particle/class_3/efficiency": 0.7102739810943604, "metrics/particle/class_3/pid_accuracy": 0.9578953981399536, "metrics/particle/class_4/efficiency": 0.7083734273910522, "metrics/particle/class_4/pid_accuracy": 0.35326087474823, "metrics/particle/class_5/efficiency": 0.8619999885559082, "metrics/particle/class_5/pid_accuracy": 0.40371230244636536, "metrics/particle/count/bias_mean": -3.1165363788604736, "metrics/particle/count/mae": 4.551432132720947, "metrics/particle/count/prediction_mean": 66.939453125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07492318004369736, "metrics/particle/event/energy_response_mean": 0.9756723046302795, "metrics/particle/event/met_abs_error": 7.8509063720703125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06122690811753273, "metrics/particle/event/scalar_pt_response_mean": 0.9884940981864929, "metrics/particle/event/vector_pt_closure": 0.06606381386518478, "metrics/particle/matched/delta_eta_abs_mean": 0.01656811125576496, "metrics/particle/matched/delta_phi_abs_mean": 0.014626863412559032, "metrics/particle/matched/delta_r_mean": 0.0247514545917511, "metrics/particle/matched/energy_relative_abs_mean": 0.15788297355175018, "metrics/particle/matched/pid_accuracy": 0.938377857208252, "metrics/particle/matched/pt_relative_abs_mean": 0.15667402744293213, "metrics/particle/matching/duplicate_fraction": 0.06686507165431976, "metrics/particle/matching/efficiency": 0.7569094896316528, "metrics/particle/matching/f1": 0.7741285562515259, "metrics/particle/matching/purity": 0.7921493053436279, "metrics/particle/matching/target_coverage_dr0p05": 0.7057041525840759, "metrics/particle/matching/target_coverage_dr0p10": 0.8136906623840332, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5410014390945435, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7977808117866516, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02295214869081974, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1535670906305313, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15309007465839386, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005956584122031927, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19320449233055115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00839298963546753, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.783211886882782, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07761779427528381, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1134891510009766, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8102.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2933212220668793, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5004234313964844, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5088996887207031, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3249077796936035, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1562839299440384, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 26.046545028686523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019449107348918915, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2073765993118286, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07587522268295288, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.208062171936035, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10129.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.740855872631073, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.5340640544891357, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.930139541625977, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.4822492599487305, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2860206663608551, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7799841165542603, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.518714904785156, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.60910964012146, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.032360076904297, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023385504260659218, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16551020741462708, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16213633120059967, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018797002732753754, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20425376296043396, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02576754428446293, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.569860577583313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07207175344228745, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.8465425968170166, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1824.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5047082901000977, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3058465719223022, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.707191228866577, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.5316550731658936, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21044176816940308, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.042713165283203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06265060603618622, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7503297328948975, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06850388646125793, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.916740894317627, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2490.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6037735939025879, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.806673526763916, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.322245597839355, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5503736734390259, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0073169805109500885, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.14983613789081573, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.143795013427734, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5905660390853882, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.839895248413086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025548744946718216, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1570863127708435, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15557457506656647, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006461597513407469, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19594763219356537, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005945656448602676, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2659957706928253, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.039897527545690536, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7728526592254639, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16819.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2359231859445572, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3544813394546509, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0766088962554932, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.890880286693573, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1499422788619995, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.517619132995605, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005002886056900024, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4939538538455963, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.028410933911800385, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8883106708526611, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20788.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7845780849456787, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.08233115077018738, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.275554656982422, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.710580825805664, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.1887046098709106, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.1414923667907715, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.474695682525635, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5976914763450623, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.223392963409424, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024614611640572548, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15659472346305847, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15526945888996124, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007149881683290005, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1957007795572281, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0046737706288695335, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5116018056869507, "metrics/particle/momentum/all/accepted/pz_bias": 0.053518593311309814, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6546475887298584, "metrics/particle/momentum/all/accepted/samples": 26745.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2733602225780487, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3951059579849243, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.329215407371521, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1447728872299194, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15637440979480743, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.847753524780273, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013679767958819866, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.87844318151474, "metrics/particle/momentum/all/assigned/pz_bias": 0.03557853028178215, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1639742851257324, "metrics/particle/momentum/all/assigned/samples": 33407.0, "metrics/particle/momentum/all/efficiency": 0.7556365728378296, "metrics/particle/momentum/all/event/energy_bias": -7.423068046569824, "metrics/particle/momentum/all/event/p3_error_mean": 21.85541343688965, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.161194801330566, "metrics/particle/momentum/all/event/px_bias": -0.8953670263290405, "metrics/particle/momentum/all/event/py_bias": 4.071312427520752, "metrics/particle/momentum/all/event/pz_abs_mean": 15.838449478149414, "metrics/particle/momentum/all/event/pz_bias": -0.3837648630142212, "metrics/particle/momentum/all/event/pz_residual_rms": 25.77094268798828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026466285809874535, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14563439786434174, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14488501846790314, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011372354812920094, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18385553359985352, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004796329885721207, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5478156805038452, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0657171905040741, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.669043779373169, "metrics/particle/momentum/class_1/accepted/samples": 14386.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13730749487876892, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3635029196739197, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.40826404094696045, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25313276052474976, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08695385605096817, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.032506227493286, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008701513521373272, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7269958853721619, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.08810855448246002, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.711962938308716, "metrics/particle/momentum/class_1/assigned/samples": 16319.0, "metrics/particle/momentum/class_1/efficiency": 0.8621082305908203, "metrics/particle/momentum/class_1/event/energy_bias": 10.206210136413574, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.410898208618164, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.405853271484375, "metrics/particle/momentum/class_1/event/px_bias": -0.2653904855251312, "metrics/particle/momentum/class_1/event/py_bias": 2.451791286468506, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.99960708618164, "metrics/particle/momentum/class_1/event/pz_bias": 1.2397006750106812, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.480253219604492, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03368566930294037, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.200312539935112, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20275679230690002, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.042933955788612366, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24316352605819702, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.027004219591617584, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8330559730529785, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.23050519824028015, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.236064434051514, "metrics/particle/momentum/class_2/accepted/samples": 1185.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7934337854385376, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8862524032592773, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0803072452545166, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6481708288192749, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3491847813129425, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.2273077964782715, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05615942180156708, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8361270427703857, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.054017260670661926, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.920602798461914, "metrics/particle/momentum/class_2/assigned/samples": 2208.0, "metrics/particle/momentum/class_2/efficiency": 0.39805173873901367, "metrics/particle/momentum/class_2/event/energy_bias": -15.92763900756836, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.700302124023438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.025031089782715, "metrics/particle/momentum/class_2/event/px_bias": 0.18219687044620514, "metrics/particle/momentum/class_2/event/py_bias": 0.6426743865013123, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.507768630981445, "metrics/particle/momentum/class_2/event/pz_bias": -0.9183185696601868, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.71436882019043, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02114100754261017, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1660120189189911, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16358572244644165, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.005920108407735825, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20456933975219727, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018558040028437972, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29138004779815674, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008502915501594543, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9334402680397034, "metrics/particle/momentum/class_3/accepted/samples": 10777.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3487498164176941, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.613450527191162, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3981027603149414, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2164604663848877, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20491233468055725, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.325468063354492, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011828555725514889, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7170104384422302, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.032645806670188904, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6226353645324707, "metrics/particle/momentum/class_3/assigned/samples": 14372.0, "metrics/particle/momentum/class_3/efficiency": 0.7086867690086365, "metrics/particle/momentum/class_3/event/energy_bias": 0.22781099379062653, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.887466430664062, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.5590362548828125, "metrics/particle/momentum/class_3/event/px_bias": -0.8249689936637878, "metrics/particle/momentum/class_3/event/py_bias": 0.24373190104961395, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.513059616088867, "metrics/particle/momentum/class_3/event/pz_bias": -1.4547233581542969, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.133256912231445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027229055762290955, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1669023633003235, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16337518393993378, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.049319203943014145, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2068648785352707, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0037878789007663727, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9720007181167603, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4021725654602051, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0473906993865967, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3200785517692566, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.1833479404449463, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4290335178375244, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.324275016784668, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19142857193946838, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.639790534973145, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022857142612338066, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3843923807144165, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.02059180662035942, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.9854025840759277, "metrics/particle/momentum/class_4/assigned/samples": 350.0, "metrics/particle/momentum/class_4/efficiency": 0.7252747416496277, "metrics/particle/momentum/class_4/event/energy_bias": -1.1016331911087036, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.062964916229248, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.091075897216797, "metrics/particle/momentum/class_4/event/px_bias": 0.004242829978466034, "metrics/particle/momentum/class_4/event/py_bias": 0.21113230288028717, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8293527364730835, "metrics/particle/momentum/class_4/event/pz_bias": 0.4703909158706665, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.087931156158447, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.019783683121204376, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16906240582466125, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1654435247182846, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012604299001395702, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20603929460048676, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02255639061331749, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7513710260391235, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.11269970238208771, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.230694055557251, "metrics/particle/momentum/class_5/accepted/samples": 133.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0965985506772995, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.27069488167762756, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.26343777775764465, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08186367154121399, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.13924050331115723, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.39727601408958435, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08227848261594772, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.7261202335357666, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5913724303245544, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.181509971618652, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.8364779949188232, "metrics/particle/momentum/class_5/event/energy_bias": -0.8278181552886963, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.797877788543701, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.067749500274658, "metrics/particle/momentum/class_5/event/px_bias": 0.008552797138690948, "metrics/particle/momentum/class_5/event/py_bias": 0.5219827890396118, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5069752931594849, "metrics/particle/momentum/class_5/event/pz_bias": 0.2791852355003357, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.606699466705322, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41027581691741943, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4115201532840729, "metrics/set_slot/angular_recall_all_dr0p10": 0.9433767795562744, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8031986951828003, "metrics/set_slot/soft_count_bias_mean": 0.06312213093042374}, "step_train_time": 4.483426094055176, "step_valid_time": 2263.7907214164734, "step_total_time": 2268.2741482257843, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.997871940584928, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16499891334694516, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3003, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2845, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2814, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9398744631648497, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473859473859474, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9436152570480929, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052614052614052616, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.929633300297324, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370629370629371, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9333333333333333, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9398744631648497, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933388848135575, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15561088817837665, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2098, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2092, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9274977895667551, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9370254577936579, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9322372806043101, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06297454220634212, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9248452696728559, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9343456900401965, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.929571206398578, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9274977895667551, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9891291467748865, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15945752253552747, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1265, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1172, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9153726708074534, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9320158102766798, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9236192714453584, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06798418972332015, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9099378881987578, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9264822134387352, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.918135526831179, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9153726708074534} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_37000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..29b513ab0b8a7f08b26ff6ab124efef8d06cc964 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7528898995641462, "train": {"Classification_binary": 0.20035865902900696, "Cardinality": 0.1319071501493454, "Classification": 0.08320870995521545, "Regression_pt": 0.17447540163993835, "Regression_eta": 0.0017296918667852879, "Regression_sin_phi": 0.0009311410831287503, "Regression_cos_phi": 0.0009747766307555139, "Regression_energy": 0.1739487648010254, "HitGrouping": 0.22724243998527527, "Auxiliary": 0.2666650116443634, "Total": 1.2518706321716309, "Total_unweighted": 0.6356271450058557}, "train_group": {}, "valid": {"Classification_binary": 0.1885142723719279, "Cardinality": 0.14484494924545288, "Classification": 0.09497034549713135, "Regression_pt": 0.1712732712427775, "Regression_eta": 0.0017630400446554024, "Regression_sin_phi": 0.0009611479472368956, "Regression_cos_phi": 0.0010001699750622113, "Regression_energy": 0.17040197054545084, "HitGrouping": 0.22504597902297974, "Auxiliary": 0.26809873183568317, "Total": 1.2612133820851643, "Total_unweighted": 0.6288842176242421, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7492250204086304, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3769413232803345, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4323226511478424, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3697115480899811, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36793753504753113, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5753282308578491, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6972042322158813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3321535587310791, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42121315002441406, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3256755769252777, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34673669934272766, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6428329348564148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.768743634223938, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3672632575035095, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4047619104385376, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3599393367767334, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.35111185908317566, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5348621010780334, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7560458183288574, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3672283887863159, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4151063859462738, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.36001065373420715, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3561337888240814, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.557568371295929, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02571357600390911, "metrics/hit_grouping/calorimeter/background_recall": 0.6308474540710449, "metrics/hit_grouping/calorimeter/completeness": 0.42473387718200684, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44854047894477844, "metrics/hit_grouping/calorimeter/energy_purity": 0.4432608187198639, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4847346842288971, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7763054370880127, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3817760944366455, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34487906098365784, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37722745537757874, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37740054726600647, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5098835229873657, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7497143149375916, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3850661516189575, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4511159658432007, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3753937780857086, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3753732740879059, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5719311237335205, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.719761073589325, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.31784701347351074, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5262293219566345, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.307537704706192, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29106971621513367, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6435343027114868, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7433536648750305, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3933461010456085, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.501401424407959, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3854220509529114, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3829849064350128, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5904702544212341, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7417994141578674, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.47525128722190857, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6357927918434143, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.46455058455467224, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4445554316043854, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5845107436180115, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7715138792991638, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3253272771835327, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2940940260887146, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.31931450963020325, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3159524202346802, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5187491774559021, "metrics/hit_grouping/calorimeter/purity": 0.4184457063674927, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8461605906486511, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06862091273069382, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15076923370361328, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06797069311141968, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03279992192983627, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0630769208073616, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7618823051452637, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14777091145515442, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1151653379201889, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14568911492824554, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13992862403392792, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.405929297208786, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7537894248962402, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3862152695655823, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4379918575286865, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3785621225833893, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3754606246948242, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.576829731464386, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7289001941680908, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4293185770511627, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45144030451774597, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45376160740852356, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6008667349815369, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7824344635009766, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3471677005290985, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4860469698905945, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4798424541950226, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46719011664390564, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6774353384971619, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4377411901950836, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39480072259902954, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40158358216285706, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.668976902961731, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7025446891784668, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42745134234428406, "metrics/hit_grouping/tracker/all/object_completeness": 0.4203875660896301, "metrics/hit_grouping/tracker/all/object_purity": 0.42466649413108826, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6304553151130676, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009272335097193718, "metrics/hit_grouping/tracker/background_recall": 0.045785702764987946, "metrics/hit_grouping/tracker/completeness": 0.4387987554073334, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5154730677604675, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7119626402854919, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41958996653556824, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47347790002822876, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4834114611148834, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6266970634460449, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6936584711074829, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4422234892845154, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4039727449417114, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4091305136680603, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6419188976287842, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6910799145698547, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42920711636543274, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31997495889663696, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31076762080192566, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6266181468963623, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.72868812084198, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44419482350349426, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44086048007011414, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4393075704574585, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5926491618156433, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7559245824813843, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4480692446231842, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4645695686340332, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4447438716888428, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5139813423156738, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6591004729270935, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4020135700702667, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3859477937221527, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4020625650882721, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7001873254776001, "metrics/hit_grouping/tracker/purity": 0.42009809613227844, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8808269500732422, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015638574957847595, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.031031087040901184, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03421977534890175, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08340573310852051, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6731052994728088, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.214618980884552, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22205403447151184, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2290559560060501, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.573872447013855, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6816713213920593, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5076441764831543, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49587804079055786, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.49999499320983887, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7112330794334412, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4258037805557251, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5349150896072388, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5868903398513794, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5848256945610046, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05992359668016434, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05402098223567009, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.84673309326172, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28015124797821045, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14971143007278442, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8431539535522461, "metrics/particle/class_1/pid_accuracy": 0.9625701308250427, "metrics/particle/class_2/efficiency": 0.3973524868488312, "metrics/particle/class_2/pid_accuracy": 0.605872392654419, "metrics/particle/class_3/efficiency": 0.6849591135978699, "metrics/particle/class_3/pid_accuracy": 0.9602230191230774, "metrics/particle/class_4/efficiency": 0.6785370707511902, "metrics/particle/class_4/pid_accuracy": 0.3957446813583374, "metrics/particle/class_5/efficiency": 0.8600000143051147, "metrics/particle/class_5/pid_accuracy": 0.42790699005126953, "metrics/particle/count/bias_mean": -5.015625, "metrics/particle/count/mae": 5.669270992279053, "metrics/particle/count/prediction_mean": 65.04036712646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07749231904745102, "metrics/particle/event/energy_response_mean": 0.9620187282562256, "metrics/particle/event/met_abs_error": 7.952459335327148, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06165497004985809, "metrics/particle/event/scalar_pt_response_mean": 0.9806298613548279, "metrics/particle/event/vector_pt_closure": 0.0636359453201294, "metrics/particle/matched/delta_eta_abs_mean": 0.014984989538788795, "metrics/particle/matched/delta_phi_abs_mean": 0.0151548832654953, "metrics/particle/matched/delta_r_mean": 0.02384229004383087, "metrics/particle/matched/energy_relative_abs_mean": 0.16190780699253082, "metrics/particle/matched/pid_accuracy": 0.9377288222312927, "metrics/particle/matched/pt_relative_abs_mean": 0.16085968911647797, "metrics/particle/matching/duplicate_fraction": 0.057386238127946854, "metrics/particle/matching/efficiency": 0.7361857295036316, "metrics/particle/matching/f1": 0.7635175585746765, "metrics/particle/matching/purity": 0.7929571270942688, "metrics/particle/matching/target_coverage_dr0p05": 0.6956861019134521, "metrics/particle/matching/target_coverage_dr0p10": 0.8003736138343811, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5153151154518127, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7756816744804382, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022915469482541084, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1577795296907425, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.157108336687088, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018000906333327293, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19813284277915955, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007923531346023083, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8036472201347351, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01926376484334469, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.116581916809082, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7951.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2567858099937439, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8885423541069031, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0039653778076172, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8317633271217346, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15111202001571655, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.283366203308105, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018076570704579353, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1925407648086548, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.024865280836820602, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3416271209716797, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9847.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7270482778549194, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.890994071960449, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.673081398010254, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.38593864440918, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1575581282377243, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.7070144414901733, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.248408317565918, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.40846776962280273, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.237197875976562, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023147815838456154, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16522769629955292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1647776961326599, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.033275481313467026, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20532841980457306, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028927963227033615, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6617639064788818, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.03949287161231041, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.056002616882324, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1763.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.42002230882644653, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6386981010437012, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.865032196044922, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.697195291519165, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1901736557483673, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 59.43035125732422, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.056755613535642624, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.645004987716675, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.26255425810813904, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.422582626342773, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2361.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5835815668106079, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.9325103759765625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.16219711303711, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5205175876617432, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.029785484075546265, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15476563572883606, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.983298301696777, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.0402731895446777, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.790443420410156, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02425343543291092, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16320760548114777, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.162042498588562, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.026929320767521858, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20260420441627502, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009184985538013279, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2722127139568329, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004240528680384159, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7877915501594543, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16331.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23466990888118744, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.267086148262024, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9778048992156982, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8005555868148804, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15787410736083984, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.648833274841309, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004898359067738056, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4972112476825714, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.008226314559578896, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9330735206604004, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20415.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7618137001991272, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9847009181976318, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.800102233886719, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.314922332763672, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.9720913171768188, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.1090739518404007, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.349985599517822, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.020797576755285263, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.749454498291016, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02377014420926571, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16168726980686188, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16072136163711548, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.024633236229419708, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20143651962280273, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00495296623557806, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.528508186340332, "metrics/particle/momentum/all/accepted/pz_bias": 0.005866474471986294, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6942133903503418, "metrics/particle/momentum/all/accepted/samples": 26045.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2547597885131836, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1797200441360474, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1946563720703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0196118354797363, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15817061066627502, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.436105728149414, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012629127129912376, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8625316619873047, "metrics/particle/momentum/all/assigned/pz_bias": -0.006348344497382641, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.113882064819336, "metrics/particle/momentum/all/assigned/samples": 32623.0, "metrics/particle/momentum/all/efficiency": 0.7358591556549072, "metrics/particle/momentum/all/event/energy_bias": -9.808205604553223, "metrics/particle/momentum/all/event/p3_error_mean": 21.753522872924805, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.72097396850586, "metrics/particle/momentum/all/event/px_bias": 1.0998640060424805, "metrics/particle/momentum/all/event/py_bias": -0.7527062296867371, "metrics/particle/momentum/all/event/pz_abs_mean": 16.331043243408203, "metrics/particle/momentum/all/event/pz_bias": -1.6526029109954834, "metrics/particle/momentum/all/event/pz_residual_rms": 25.754724502563477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025221409276127815, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15380151569843292, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15269838273525238, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02714851312339306, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1921338438987732, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0047537959180772305, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5657274127006531, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007446266710758209, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7006754875183105, "metrics/particle/momentum/class_1/accepted/samples": 14094.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13045184314250946, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.39399492740631104, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4453054666519165, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2966180443763733, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09692231565713882, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.2691850662231445, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007818317040801048, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7291820645332336, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0037881333846598864, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.372225046157837, "metrics/particle/momentum/class_1/assigned/samples": 16116.0, "metrics/particle/momentum/class_1/efficiency": 0.8446095585823059, "metrics/particle/momentum/class_1/event/energy_bias": 5.948246002197266, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.394859313964844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.71912670135498, "metrics/particle/momentum/class_1/event/px_bias": 0.45902907848358154, "metrics/particle/momentum/class_1/event/py_bias": -0.8874431848526001, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.268901824951172, "metrics/particle/momentum/class_1/event/pz_bias": 0.36604422330856323, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.635778427124023, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03340218588709831, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20746417343616486, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21006661653518677, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.01865444704890251, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.251120924949646, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030647985637187958, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.853266954421997, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.05926874279975891, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.232037544250488, "metrics/particle/momentum/class_2/accepted/samples": 1142.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6804112792015076, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6868134140968323, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9049360752105713, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4858086109161377, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3285643756389618, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.671193599700928, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05722742900252342, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.689746618270874, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.14733153581619263, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.919209003448486, "metrics/particle/momentum/class_2/assigned/samples": 2027.0, "metrics/particle/momentum/class_2/efficiency": 0.3836076557636261, "metrics/particle/momentum/class_2/event/energy_bias": -16.500797271728516, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.791532516479492, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.83329963684082, "metrics/particle/momentum/class_2/event/px_bias": 0.2576274275779724, "metrics/particle/momentum/class_2/event/py_bias": 0.5853209495544434, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.738083839416504, "metrics/particle/momentum/class_2/event/pz_bias": -3.4102931022644043, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.68396759033203, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02078498899936676, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1666399985551834, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16560068726539612, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02531474642455578, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20693939924240112, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019201228860765696, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.30293935537338257, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.007884367369115353, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0041124820709229, "metrics/particle/momentum/class_3/accepted/samples": 10416.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33777371048927307, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.1038296222686768, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.057699203491211, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8864591121673584, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20383213460445404, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.121484756469727, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010867234319448471, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7248228192329407, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.005786399822682142, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.242702007293701, "metrics/particle/momentum/class_3/assigned/samples": 13987.0, "metrics/particle/momentum/class_3/efficiency": 0.6849477291107178, "metrics/particle/momentum/class_3/event/energy_bias": 2.269498109817505, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.754148483276367, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.060521602630615, "metrics/particle/momentum/class_3/event/px_bias": 0.5159181952476501, "metrics/particle/momentum/class_3/event/py_bias": -0.4187060594558716, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.094494819641113, "metrics/particle/momentum/class_3/event/pz_bias": 0.6972038149833679, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.412578582763672, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02571760304272175, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17061613500118256, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16500161588191986, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05076460540294647, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2051735669374466, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007905138656497002, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9063592553138733, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3239738345146179, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.041849374771118, "metrics/particle/momentum/class_4/accepted/samples": 253.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27699971199035645, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.795682668685913, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.388640880584717, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.269346237182617, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2023809552192688, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 30.27799415588379, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02083333395421505, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1951289176940918, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.361397922039032, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4676084518432617, "metrics/particle/momentum/class_4/assigned/samples": 336.0, "metrics/particle/momentum/class_4/efficiency": 0.6950549483299255, "metrics/particle/momentum/class_4/event/energy_bias": -0.7087615132331848, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0031542778015137, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0231451988220215, "metrics/particle/momentum/class_4/event/px_bias": -0.07678443938493729, "metrics/particle/momentum/class_4/event/py_bias": -0.3637929856777191, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8597443103790283, "metrics/particle/momentum/class_4/event/pz_bias": 0.6161227226257324, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.627660751342773, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017674921080470085, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19753247499465942, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1951313018798828, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.026593415066599846, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23623351752758026, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0748343467712402, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.14058630168437958, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.7649102210998535, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07618042081594467, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2713145315647125, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.27261409163475037, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08509334921836853, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08280254900455475, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4922296106815338, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07006369531154633, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5165483951568604, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0637316182255745, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.717456340789795, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.8805031180381775, "metrics/particle/momentum/class_5/event/energy_bias": -0.8163906335830688, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7602553367614746, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0312881469726562, "metrics/particle/momentum/class_5/event/px_bias": -0.05592622607946396, "metrics/particle/momentum/class_5/event/py_bias": 0.33191508054733276, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5504463911056519, "metrics/particle/momentum/class_5/event/pz_bias": 0.07831967622041702, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.904202938079834, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41077837347984314, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4118718206882477, "metrics/set_slot/angular_recall_all_dr0p10": 0.9442131519317627, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7961266040802002, "metrics/set_slot/soft_count_bias_mean": -1.3660821914672852}, "step_train_time": 4.559496641159058, "step_valid_time": 4735.649401187897, "step_total_time": 4740.208898067474, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974550429322697, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16650446041324407, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3007, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2845, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2815, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9398744631648497, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9461257066844031, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9429897248922771, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05387429331559694, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299636603898249, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361489857000332, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9330460722572091, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9398744631648497, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9869882116144575, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16597777789875323, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2114, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372236958443855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9502465262214254, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.943690184731805, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04975347377857463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9345711759504863, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9475571492604213, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9410193634542622, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372236958443855, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9906057814663705, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1544757911430551, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1273, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1182, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9177018633540373, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9285153181461115, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9230769230769231, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07148468185388845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9130434782608695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9238020424194815, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9183912534166341, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9177018633540373} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_38000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..da544a2f2719ce370f631b281f663d81d91c0ded --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.8002653022550692, "train": {"Classification_binary": 0.1891782283782959, "Cardinality": 0.12410181015729904, "Classification": 0.09046000242233276, "Regression_pt": 0.1607666015625, "Regression_eta": 0.0022072051651775837, "Regression_sin_phi": 0.0010457364842295647, "Regression_cos_phi": 0.0009316559880971909, "Regression_energy": 0.15971507132053375, "HitGrouping": 0.2218799740076065, "Auxiliary": 0.256264328956604, "Total": 1.2294118404388428, "Total_unweighted": 0.6043045013211668}, "train_group": {}, "valid": {"Classification_binary": 0.18619434038798013, "Cardinality": 0.13154671589533487, "Classification": 0.09422615170478821, "Regression_pt": 0.1757438580195109, "Regression_eta": 0.0017601482880612214, "Regression_sin_phi": 0.0009574661186585823, "Regression_cos_phi": 0.000953059953947862, "Regression_energy": 0.17495439449946085, "HitGrouping": 0.22643830378850302, "Auxiliary": 0.26940494775772095, "Total": 1.2482916514078777, "Total_unweighted": 0.6347894189724077, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.75202476978302, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3739977777004242, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4323226511478424, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.36531928181648254, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36316022276878357, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5617266297340393, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6990104913711548, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.34596318006515503, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4235852360725403, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3373788297176361, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.36048662662506104, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6296170949935913, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7640727758407593, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3633021116256714, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4065179228782654, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35602006316185, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.35066407918930054, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5421960353851318, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7543599605560303, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3651886582374573, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4163525700569153, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35734042525291443, "metrics/hit_grouping/calorimeter/all/object_purity": 0.35556626319885254, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5563222169876099, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02563929185271263, "metrics/hit_grouping/calorimeter/background_recall": 0.6305204629898071, "metrics/hit_grouping/calorimeter/completeness": 0.4188213348388672, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44401293992996216, "metrics/hit_grouping/calorimeter/energy_purity": 0.43875110149383545, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.48895063996315, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7762277126312256, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.381656676530838, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3445804715156555, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37629470229148865, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3755767047405243, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5052851438522339, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7525991201400757, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3790484368801117, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4543910622596741, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.369038850069046, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3700089454650879, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5690199136734009, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.709908127784729, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3158881962299347, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5286310315132141, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3050295114517212, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2981601655483246, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6511186957359314, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7367725968360901, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3887852430343628, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5052008628845215, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.38042107224464417, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.37959930300712585, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5978822708129883, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7291212677955627, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4731665551662445, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.633176326751709, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4588831961154938, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4638839364051819, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.599162757396698, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7764971256256104, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32600370049476624, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2930895984172821, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3195335865020752, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3158685863018036, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5061604380607605, "metrics/hit_grouping/calorimeter/purity": 0.41258925199508667, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8562667369842529, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06725204735994339, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1523076891899109, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06637362390756607, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.029522931203246117, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7623058557510376, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1458030492067337, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11972633749246597, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14299872517585754, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1350289136171341, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.40079817175865173, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7517308592796326, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38415706157684326, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4390411972999573, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3758702278137207, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3752000033855438, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5761083364486694, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7285110354423523, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.437238484621048, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4448566138744354, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4459899663925171, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5968320369720459, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7765370607376099, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.33875492215156555, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4778292775154114, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4798874855041504, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47223779559135437, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6777628064155579, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.437898725271225, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3908056318759918, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.39315569400787354, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6714972257614136, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7021214365959167, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4293406307697296, "metrics/hit_grouping/tracker/all/object_completeness": 0.41521862149238586, "metrics/hit_grouping/tracker/all/object_purity": 0.4171595871448517, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6311165690422058, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0047886730171740055, "metrics/hit_grouping/tracker/background_recall": 0.02754249796271324, "metrics/hit_grouping/tracker/completeness": 0.4352325201034546, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5195990800857544, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7099952697753906, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42177677154541016, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47011762857437134, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4783611595630646, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6266059279441833, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6955702304840088, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4454597234725952, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.393793523311615, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3958914279937744, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6461069583892822, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6916031241416931, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42900484800338745, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3161146342754364, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.30389782786369324, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6252022385597229, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7297279238700867, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4460819661617279, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4342138469219208, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4301626682281494, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5912113785743713, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7542281150817871, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46338215470314026, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46888694167137146, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43436700105667114, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5332889556884766, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6569951772689819, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40154531598091125, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3810358941555023, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39719417691230774, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.700304388999939, "metrics/hit_grouping/tracker/purity": 0.414806604385376, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.887939453125, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.016507385298609734, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02811589650809765, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.029486555606126785, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0842745453119278, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6759976148605347, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20528770983219147, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20255602896213531, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20781514048576355, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5635043978691101, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.679846465587616, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5109497904777527, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4920344650745392, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.49367502331733704, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7131220102310181, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.42836833000183105, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5397254228591919, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5970602035522461, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6033453941345215, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.057359032332897186, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07254062592983246, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.847091674804688, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2868407070636749, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1564009040594101, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8536010980606079, "metrics/particle/class_1/pid_accuracy": 0.9583922624588013, "metrics/particle/class_2/efficiency": 0.3948844373226166, "metrics/particle/class_2/pid_accuracy": 0.5977272987365723, "metrics/particle/class_3/efficiency": 0.697659432888031, "metrics/particle/class_3/pid_accuracy": 0.9606088399887085, "metrics/particle/class_4/efficiency": 0.6929740309715271, "metrics/particle/class_4/pid_accuracy": 0.34583333134651184, "metrics/particle/class_5/efficiency": 0.8759999871253967, "metrics/particle/class_5/pid_accuracy": 0.422374427318573, "metrics/particle/count/bias_mean": -3.3919270038604736, "metrics/particle/count/mae": 4.622395992279053, "metrics/particle/count/prediction_mean": 66.6640625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08037414401769638, "metrics/particle/event/energy_response_mean": 0.9558669924736023, "metrics/particle/event/met_abs_error": 7.6893744468688965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06332730501890182, "metrics/particle/event/scalar_pt_response_mean": 0.9781382083892822, "metrics/particle/event/vector_pt_closure": 0.06421693414449692, "metrics/particle/matched/delta_eta_abs_mean": 0.016065917909145355, "metrics/particle/matched/delta_phi_abs_mean": 0.014850429259240627, "metrics/particle/matched/delta_r_mean": 0.024475019425153732, "metrics/particle/matched/energy_relative_abs_mean": 0.16091060638427734, "metrics/particle/matched/pid_accuracy": 0.9350751042366028, "metrics/particle/matched/pt_relative_abs_mean": 0.16161495447158813, "metrics/particle/matching/duplicate_fraction": 0.06496347486972809, "metrics/particle/matching/efficiency": 0.7466033697128296, "metrics/particle/matching/f1": 0.7651260495185852, "metrics/particle/matching/purity": 0.7845911979675293, "metrics/particle/matching/target_coverage_dr0p05": 0.7031392455101013, "metrics/particle/matching/target_coverage_dr0p10": 0.811376690864563, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5202590823173523, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.787734866142273, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02381851337850094, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15736179053783417, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15870946645736694, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010166855528950691, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19893676042556763, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00956284161657095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8321691751480103, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04974224790930748, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2189652919769287, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8052.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2848646640777588, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1155823469161987, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.358502984046936, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1763805150985718, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15892203152179718, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.282025337219238, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02011295035481453, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2386951446533203, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0548873245716095, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2886078357696533, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10093.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7362838387489319, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.5563366413116455, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.605411529541016, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.431229114532471, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5274221897125244, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5756417512893677, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.184608459472656, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.20235924422740936, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.595009803771973, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023742439225316048, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1563912034034729, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15711364150047302, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0008773545268923044, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19747847318649292, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030352303758263588, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5654542446136475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08922954648733139, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.784421443939209, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1845.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.49050262570381165, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9567530155181885, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1476645469665527, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9589484930038452, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19305330514907837, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.295039176940918, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06058158352971077, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.609402656555176, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.037525687366724014, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.958947658538818, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2476.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.610724925994873, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.208511352539062, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.953062057495117, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4322038888931274, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05616913363337517, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06427165120840073, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.795116424560547, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.440497636795044, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.68272590637207, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02456769160926342, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16114698350429535, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16146497428417206, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015375157818198204, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2024056613445282, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007274931995198131, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2763088047504425, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0007166302530094981, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8021073937416077, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16495.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2460845708847046, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3682761192321777, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0804468393325806, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.897933304309845, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16284005343914032, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.934377670288086, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004582288209348917, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5002318024635315, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.011548780836164951, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8347758054733276, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20732.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7694640159606934, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.242234230041504, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.713994026184082, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.232072830200195, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6084573268890381, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.105218768119812, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.295031547546387, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.48870694637298584, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.4751458168029785, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02428143471479416, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15965968370437622, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16032010316848755, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012772639282047749, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20101132988929749, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0054940893314778805, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5360186100006104, "metrics/particle/momentum/all/accepted/pz_bias": 0.021861698478460312, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7045620679855347, "metrics/particle/momentum/all/accepted/samples": 26392.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2760111689567566, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2610911130905151, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2440710067749023, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0612149238586426, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16389897465705872, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.160441398620605, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013453049585223198, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8808695077896118, "metrics/particle/momentum/all/assigned/pz_bias": 0.012235731817781925, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1754465103149414, "metrics/particle/momentum/all/assigned/samples": 33301.0, "metrics/particle/momentum/all/efficiency": 0.7456631064414978, "metrics/particle/momentum/all/event/energy_bias": -13.00708293914795, "metrics/particle/momentum/all/event/p3_error_mean": 21.559932708740234, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.4374418258667, "metrics/particle/momentum/all/event/px_bias": 0.13720431923866272, "metrics/particle/momentum/all/event/py_bias": -1.616588830947876, "metrics/particle/momentum/all/event/pz_abs_mean": 16.274341583251953, "metrics/particle/momentum/all/event/pz_bias": -0.7494312524795532, "metrics/particle/momentum/all/event/pz_residual_rms": 26.156526565551758, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025699397549033165, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15393655002117157, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15418939292430878, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012565658427774906, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1938064992427826, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005472148302942514, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.568852961063385, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.018680887296795845, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.705737829208374, "metrics/particle/momentum/class_1/accepted/samples": 14254.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.131901815533638, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3528774082660675, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3943369686603546, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2316296547651291, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09451144188642502, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6460766792297363, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00935269519686699, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7383193373680115, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03692636266350746, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.411818265914917, "metrics/particle/momentum/class_1/assigned/samples": 16252.0, "metrics/particle/momentum/class_1/efficiency": 0.8541978597640991, "metrics/particle/momentum/class_1/event/energy_bias": -0.6280040144920349, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.936033248901367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.580232620239258, "metrics/particle/momentum/class_1/event/px_bias": 0.9472307562828064, "metrics/particle/momentum/class_1/event/py_bias": -0.7490119338035583, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.840473175048828, "metrics/particle/momentum/class_1/event/pz_bias": -0.9115800857543945, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.785240173339844, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03219405561685562, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20445148646831512, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2082708179950714, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04273373261094093, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24734485149383545, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03690987080335617, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9460396766662598, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1351979672908783, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.398835182189941, "metrics/particle/momentum/class_2/accepted/samples": 1165.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7753544449806213, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8123792409896851, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.112376093864441, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.658888041973114, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34712642431259155, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.764750957489014, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06390804797410965, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.850602388381958, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.029503485187888145, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.06000280380249, "metrics/particle/momentum/class_2/assigned/samples": 2175.0, "metrics/particle/momentum/class_2/efficiency": 0.39133355021476746, "metrics/particle/momentum/class_2/event/energy_bias": -15.64986801147461, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.25128936767578, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.7022705078125, "metrics/particle/momentum/class_2/event/px_bias": 0.3671460747718811, "metrics/particle/momentum/class_2/event/py_bias": -1.2664673328399658, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.295452117919922, "metrics/particle/momentum/class_2/event/pz_bias": -0.6474871635437012, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.70009422302246, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021566122770309448, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16173285245895386, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1627035290002823, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01802452653646469, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2041682004928589, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001985440030694008, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.30998197197914124, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007544540800154209, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9874669313430786, "metrics/particle/momentum/class_3/accepted/samples": 10577.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36428359150886536, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.3366453647613525, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.214294195175171, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0482354164123535, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21413663029670715, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.30847930908203, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010365938767790794, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7240901589393616, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.013411635532975197, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.2983791828155518, "metrics/particle/momentum/class_3/assigned/samples": 14374.0, "metrics/particle/momentum/class_3/efficiency": 0.6955349445343018, "metrics/particle/momentum/class_3/event/energy_bias": 5.818515300750732, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.64504337310791, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.0818939208984375, "metrics/particle/momentum/class_3/event/px_bias": -1.1163402795791626, "metrics/particle/momentum/class_3/event/py_bias": 0.14119580388069153, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.006288528442383, "metrics/particle/momentum/class_3/event/pz_bias": 0.7116785645484924, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.99339771270752, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025378286838531494, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1742083579301834, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1736229956150055, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06934701651334763, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2148428112268448, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003846153849735856, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0113704204559326, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3241841793060303, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1338870525360107, "metrics/particle/momentum/class_4/accepted/samples": 260.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.347592294216156, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.507540225982666, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1214334964752197, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.00726580619812, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.20520231127738953, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.266414642333984, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0028901733458042145, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0420323610305786, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2717900276184082, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1197214126586914, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.7142857313156128, "metrics/particle/momentum/class_4/event/energy_bias": -1.2093393802642822, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0555927753448486, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0911383628845215, "metrics/particle/momentum/class_4/event/px_bias": 0.057775065302848816, "metrics/particle/momentum/class_4/event/py_bias": 0.07602886855602264, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8588560819625854, "metrics/particle/momentum/class_4/event/pz_bias": 0.22329199314117432, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.447086811065674, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016963712871074677, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18674884736537933, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18132176995277405, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.006663825362920761, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22386795282363892, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.014705882407724857, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6867706775665283, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08011462539434433, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.9334375858306885, "metrics/particle/momentum/class_5/accepted/samples": 136.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03185480460524559, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2542768716812134, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24902164936065674, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.039893846958875656, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11688311398029327, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3433583378791809, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04545454680919647, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.3765909671783447, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.1932182013988495, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.730326175689697, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.8553459048271179, "metrics/particle/momentum/class_5/event/energy_bias": -1.3383866548538208, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.875601291656494, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.105234384536743, "metrics/particle/momentum/class_5/event/px_bias": -0.11860740929841995, "metrics/particle/momentum/class_5/event/py_bias": 0.18166564404964447, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5552709102630615, "metrics/particle/momentum/class_5/event/pz_bias": -0.12533468008041382, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.01084566116333, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41307660937309265, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.41501832008361816, "metrics/set_slot/angular_recall_all_dr0p10": 0.9432466626167297, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8018883466720581, "metrics/set_slot/soft_count_bias_mean": -0.4376302659511566}, "step_train_time": 4.609321594238281, "step_valid_time": 7196.0526757240295, "step_total_time": 7200.661997795105, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9942373696606249, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1677211745386894, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2999, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2832, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2804, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355797819623389, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9443147715905301, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9399269830733489, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05568522840946982, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9263296993723158, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9349783261087029, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9306339196813807, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355797819623389, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9810783066304081, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.157402903833877, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2230, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2105, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9350132625994695, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9484304932735426, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9416740872662511, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0515695067264574, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9305923961096375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9439461883408071, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9372217275155833, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9350132625994695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9858311797207189, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15475044986219744, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1160, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1157, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9006211180124224, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9199048374306106, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9101608473911338, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08009516256938938, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8982919254658385, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9175257731958762, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9078069831306395, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9006211180124224} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_39000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..768a9a2877617abae03fa83ab665178c1ad8f7fd --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.847640704945992, "train": {"Classification_binary": 0.18609386682510376, "Cardinality": 0.15318483114242554, "Classification": 0.08534983545541763, "Regression_pt": 0.15998877584934235, "Regression_eta": 0.001380787929520011, "Regression_sin_phi": 0.0008521260460838675, "Regression_cos_phi": 0.0009131476981565356, "Regression_energy": 0.15888549387454987, "HitGrouping": 0.2181740254163742, "Auxiliary": 0.2608708441257477, "Total": 1.2080602645874023, "Total_unweighted": 0.593464033678174}, "train_group": {}, "valid": {"Classification_binary": 0.18197967608769736, "Cardinality": 0.1254134178161621, "Classification": 0.0898396869500478, "Regression_pt": 0.16214988629023233, "Regression_eta": 0.0016583427786827087, "Regression_sin_phi": 0.0009245235317697128, "Regression_cos_phi": 0.0009492150663087765, "Regression_energy": 0.16132918993631998, "HitGrouping": 0.2199472983678182, "Auxiliary": 0.25896793603897095, "Total": 1.1987438201904297, "Total_unweighted": 0.5988305206410587, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7553991079330444, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.39354753494262695, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43591195344924927, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.387434184551239, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.38705313205718994, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5479361414909363, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.697246789932251, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.36054059863090515, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4208742678165436, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.35297641158103943, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.37425437569618225, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6367332935333252, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7725471258163452, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.38651877641677856, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40543332695961, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3797574043273926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.37308892607688904, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5178700685501099, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7602748870849609, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3864504098892212, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41662612557411194, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3798255920410156, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3776870667934418, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5382066965103149, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026064438745379448, "metrics/hit_grouping/calorimeter/background_recall": 0.6360355615615845, "metrics/hit_grouping/calorimeter/completeness": 0.4364622235298157, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4575968384742737, "metrics/hit_grouping/calorimeter/energy_purity": 0.4522778391838074, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4710122346878052, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7826902866363525, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4060409963130951, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3399820923805237, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4020053446292877, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.39775505661964417, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4833681583404541, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7576904892921448, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3973393142223358, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.45827269554138184, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3888346552848816, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.39279529452323914, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5536147356033325, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7155219912528992, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3336363434791565, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5354569554328918, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.32349008321762085, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3194654881954193, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.638225257396698, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7447111010551453, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.410048246383667, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5008408427238464, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4030577838420868, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4066362679004669, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5772656202316284, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7339040040969849, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.48169657588005066, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6336996555328369, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4693716764450073, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4713261127471924, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5960230231285095, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7803813219070435, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.34889325499534607, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29831257462501526, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.34339088201522827, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3345823884010315, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4888174533843994, "metrics/hit_grouping/calorimeter/purity": 0.43015533685684204, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8361501693725586, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08274064213037491, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.17076922953128815, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08355555683374405, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.038378361612558365, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.06615384668111801, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.76328444480896, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.15762129426002502, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12371721863746643, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.15490815043449402, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14894156157970428, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3722918927669525, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7584846019744873, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4060850739479065, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.43871328234672546, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3990766704082489, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3980756402015686, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5578108429908752, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7284476161003113, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43693962693214417, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45226508378982544, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4565705358982086, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6011655926704407, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7792977094650269, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34043747186660767, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4813365936279297, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4819900691509247, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4694335460662842, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6755375862121582, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4399464428424835, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3997480571269989, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40756335854530334, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6766165494918823, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7009994387626648, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4306159019470215, "metrics/hit_grouping/tracker/all/object_completeness": 0.4232185482978821, "metrics/hit_grouping/tracker/all/object_purity": 0.4293212592601776, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.635320246219635, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005892813205718994, "metrics/hit_grouping/tracker/background_recall": 0.03340638428926468, "metrics/hit_grouping/tracker/completeness": 0.4422106146812439, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5107128024101257, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7139558792114258, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42186787724494934, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47672680020332336, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4833877980709076, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6227790713310242, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6915821433067322, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4441271722316742, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40451881289482117, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4161252975463867, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6541024446487427, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6822436451911926, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4356796145439148, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32430586218833923, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3233219087123871, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.643203854560852, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7249780297279358, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4466211497783661, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4429594576358795, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.44702574610710144, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6008267402648926, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7444308400154114, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4520639181137085, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4657525420188904, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4485436975955963, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5605858564376831, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.662124752998352, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40599390864372253, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39002203941345215, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40287697315216064, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6933973431587219, "metrics/hit_grouping/tracker/purity": 0.42192527651786804, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8823198676109314, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0152041707187891, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03121201880276203, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.033996373414993286, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08384013921022415, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6771583557128906, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20839813351631165, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2179335355758667, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2232554703950882, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5635043978691101, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6790756583213806, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5123664736747742, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49986061453819275, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.506503164768219, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7184345722198486, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3744460940361023, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5736807584762573, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6327848434448242, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6036735773086548, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11128126829862595, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07286880910396576, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.032161712646484, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30666178464889526, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17622198164463043, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8757222294807434, "metrics/particle/class_1/pid_accuracy": 0.9638717770576477, "metrics/particle/class_2/efficiency": 0.40430784225463867, "metrics/particle/class_2/pid_accuracy": 0.599056601524353, "metrics/particle/class_3/efficiency": 0.7185978293418884, "metrics/particle/class_3/pid_accuracy": 0.9649211764335632, "metrics/particle/class_4/efficiency": 0.7179980874061584, "metrics/particle/class_4/pid_accuracy": 0.3713136613368988, "metrics/particle/class_5/efficiency": 0.8820000290870667, "metrics/particle/class_5/pid_accuracy": 0.4263038635253906, "metrics/particle/count/bias_mean": -2.666015625, "metrics/particle/count/mae": 4.241536617279053, "metrics/particle/count/prediction_mean": 67.38997650146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07767871022224426, "metrics/particle/event/energy_response_mean": 0.952781617641449, "metrics/particle/event/met_abs_error": 7.590911388397217, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06374893337488174, "metrics/particle/event/scalar_pt_response_mean": 0.9674387574195862, "metrics/particle/event/vector_pt_closure": 0.06070131063461304, "metrics/particle/matched/delta_eta_abs_mean": 0.014431937597692013, "metrics/particle/matched/delta_phi_abs_mean": 0.014615971595048904, "metrics/particle/matched/delta_r_mean": 0.022991182282567024, "metrics/particle/matched/energy_relative_abs_mean": 0.15785834193229675, "metrics/particle/matched/pid_accuracy": 0.9401424527168274, "metrics/particle/matched/pt_relative_abs_mean": 0.1563735157251358, "metrics/particle/matching/duplicate_fraction": 0.06232187896966934, "metrics/particle/matching/efficiency": 0.7671133875846863, "metrics/particle/matching/f1": 0.7819929122924805, "metrics/particle/matching/purity": 0.7974611520767212, "metrics/particle/matching/target_coverage_dr0p05": 0.7232868075370789, "metrics/particle/matching/target_coverage_dr0p10": 0.8243220448493958, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5496347546577454, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.806144654750824, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022163216024637222, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15482841432094574, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15408335626125336, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0030475992243736982, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19491013884544373, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009486743248999119, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8306176662445068, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02250126376748085, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2848846912384033, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8222.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2825429439544678, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2066757678985596, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.353519320487976, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1656162738800049, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1531020998954773, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.362996101379395, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019931606948375702, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2268898487091064, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05232018977403641, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.387592315673828, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10235.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7518288493156433, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.2941994667053223, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.149591445922852, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.22543478012085, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.46151667833328247, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.038408346474170685, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.80452823638916, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.185264229774475, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.129948616027832, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022933419793844223, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16182659566402435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16087166965007782, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0074435267597436905, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20196539163589478, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03563829883933067, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7601954936981201, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.014226766303181648, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.535460472106934, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1880.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5110251903533936, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.518460988998413, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.7006866931915283, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.509399175643921, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2007889598608017, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.07099151611328, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06666667014360428, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8355510234832764, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.052588626742362976, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.865588188171387, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2535.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6223105192184448, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.495051860809326, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.64643383026123, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3027424812316895, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.011278145015239716, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.046546854078769684, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.491990089416504, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.0677996575832367, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.04686737060547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023230677470564842, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15899895131587982, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1568862795829773, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006387867033481598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19708549976348877, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008200082229450345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26746171712875366, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00039573066169396043, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7824522852897644, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17073.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2215353399515152, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2742626667022705, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0129948854446411, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8271042108535767, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14494004845619202, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.16462516784668, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0050839330069720745, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.47448572516441345, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.015166106633841991, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.812431812286377, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20850.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7964267134666443, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.207628011703491, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.910463333129883, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.46082878112793, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.20061782002449036, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.2408156543970108, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.335997104644775, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.15853706002235413, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.188525199890137, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02288714423775673, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15793274343013763, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1563139408826828, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005450275726616383, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.196772500872612, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005850966088473797, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5411179065704346, "metrics/particle/momentum/all/accepted/pz_bias": 0.007543529849499464, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.840458631515503, "metrics/particle/momentum/all/accepted/samples": 27175.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2619359493255615, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2721000909805298, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2439159154891968, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0570056438446045, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15163592994213104, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.180651664733887, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0142474714666605, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8815695643424988, "metrics/particle/momentum/all/assigned/pz_bias": 0.029298707842826843, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1930689811706543, "metrics/particle/momentum/all/assigned/samples": 33620.0, "metrics/particle/momentum/all/efficiency": 0.7677854895591736, "metrics/particle/momentum/all/event/energy_bias": -12.996879577636719, "metrics/particle/momentum/all/event/p3_error_mean": 20.938800811767578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.844059944152832, "metrics/particle/momentum/all/event/px_bias": 0.6508563756942749, "metrics/particle/momentum/all/event/py_bias": 0.23267719149589539, "metrics/particle/momentum/all/event/pz_abs_mean": 15.28824520111084, "metrics/particle/momentum/all/event/pz_bias": 1.2760014533996582, "metrics/particle/momentum/all/event/pz_residual_rms": 23.446243286132812, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023998042568564415, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14723551273345947, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14616793394088745, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0034753496292978525, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1851174384355545, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00615384615957737, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5792268514633179, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01663789339363575, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8463125228881836, "metrics/particle/momentum/class_1/accepted/samples": 14625.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1221533715724945, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3414613604545593, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39496463537216187, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.22195453941822052, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07635829597711563, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.283221244812012, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010034263134002686, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7343135476112366, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04059246554970741, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.480020046234131, "metrics/particle/momentum/class_1/assigned/samples": 16344.0, "metrics/particle/momentum/class_1/efficiency": 0.8764307498931885, "metrics/particle/momentum/class_1/event/energy_bias": -0.25695163011550903, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.944377899169922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.4160737991333, "metrics/particle/momentum/class_1/event/px_bias": -0.15975934267044067, "metrics/particle/momentum/class_1/event/py_bias": 1.1257623434066772, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.046564102172852, "metrics/particle/momentum/class_1/event/pz_bias": 0.17510363459587097, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.975528717041016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03348655626177788, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2045827955007553, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2037539929151535, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03963290527462959, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2448987066745758, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03294892981648445, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8970410823822021, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08121839910745621, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.682415962219238, "metrics/particle/momentum/class_2/accepted/samples": 1214.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7912187576293945, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8288933634757996, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9635921716690063, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5346689820289612, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3485254645347595, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3926584720611572, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06478998810052872, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8935129642486572, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0605587437748909, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.599903583526611, "metrics/particle/momentum/class_2/assigned/samples": 2238.0, "metrics/particle/momentum/class_2/efficiency": 0.4077930748462677, "metrics/particle/momentum/class_2/event/energy_bias": -15.644244194030762, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.346694946289062, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.44985580444336, "metrics/particle/momentum/class_2/event/px_bias": 0.6503125429153442, "metrics/particle/momentum/class_2/event/py_bias": -1.2013792991638184, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.521238327026367, "metrics/particle/momentum/class_2/event/pz_bias": 0.9421291947364807, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.515506744384766, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020274825394153595, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16615323722362518, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16397370398044586, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021887483075261116, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2050274759531021, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020133613143116236, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3096260726451874, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.019550641998648643, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0551574230194092, "metrics/particle/momentum/class_3/accepted/samples": 10927.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3400183916091919, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.3528006076812744, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2225446701049805, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0561912059783936, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2062203288078308, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.77206039428711, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010734191164374352, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7130178213119507, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.005779816769063473, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.065967559814453, "metrics/particle/momentum/class_3/assigned/samples": 14533.0, "metrics/particle/momentum/class_3/efficiency": 0.7185506820678711, "metrics/particle/momentum/class_3/event/energy_bias": 5.652596950531006, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.332138061523438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.826577186584473, "metrics/particle/momentum/class_3/event/px_bias": 0.31494075059890747, "metrics/particle/momentum/class_3/event/py_bias": -0.4056282341480255, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.745414733886719, "metrics/particle/momentum/class_3/event/pz_bias": -0.4061051905155182, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.320999145507812, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023996463045477867, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1818419247865677, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17117026448249817, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03253885358572006, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2116447538137436, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01498127356171608, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.044379711151123, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.30743110179901123, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6505651473999023, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2540593445301056, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.0933492183685303, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4301199913024902, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2961502075195312, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17478510737419128, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.327159881591797, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017191976308822632, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1952260732650757, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.336342990398407, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6523489952087402, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.7335164546966553, "metrics/particle/momentum/class_4/event/energy_bias": -1.6843451261520386, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.059684991836548, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.101501226425171, "metrics/particle/momentum/class_4/event/px_bias": -0.054765112698078156, "metrics/particle/momentum/class_4/event/py_bias": 0.1379244029521942, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.822891354560852, "metrics/particle/momentum/class_4/event/pz_bias": 0.48225897550582886, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.168569564819336, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0167886670678854, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18331775069236755, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17834359407424927, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.005632395390421152, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22080904245376587, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0211267601698637, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.891163945198059, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.037937723100185394, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.2893447875976562, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0571243092417717, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.379902720451355, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3864094913005829, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.18163040280342102, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07692307978868484, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.12536883354187, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.4464304447174072, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.09828732907772064, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.73856782913208, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.893081784248352, "metrics/particle/momentum/class_5/event/energy_bias": -1.0639358758926392, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.745614767074585, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9890432357788086, "metrics/particle/momentum/class_5/event/px_bias": -0.09987242519855499, "metrics/particle/momentum/class_5/event/py_bias": 0.5759978294372559, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5200510025024414, "metrics/particle/momentum/class_5/event/pz_bias": 0.08261492103338242, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.972439289093018, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39226147532463074, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3936692178249359, "metrics/set_slot/angular_recall_all_dr0p10": 0.9462855458259583, "metrics/set_slot/angular_recall_selected_dr0p10": 0.813226044178009, "metrics/set_slot/soft_count_bias_mean": 0.41490957140922546}, "step_train_time": 4.325433015823364, "step_valid_time": 9671.260733366013, "step_total_time": 9675.586166858673, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9804590577930915, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15753952744719424, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2997, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2843, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.939213742979848, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9486152819486153, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9438911022576362, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05138471805138472, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9286422200198216, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9379379379379379, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9332669322709163, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.939213742979848, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.973486345150951, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15274148655941888, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2235, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2111, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372236958443855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9485458612975392, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9428507894151656, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05145413870246085, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9332449160035367, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9445190156599552, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9388481209695353, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372236958443855, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9699651280038475, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14905478450091136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1266, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9184782608695652, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9344391785150079, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9263899765074393, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06556082148499211, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9130434782608695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9289099526066351, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9209083790133125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9184782608695652} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_4000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..0f6c4df228f3d1976f90c2d2d37813889f2efc18 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.18950161076369149, "train": {"Classification_binary": 0.3612162470817566, "Cardinality": 0.18907274305820465, "Classification": 0.2583920359611511, "Regression_pt": 0.4536212682723999, "Regression_eta": 0.0046465350314974785, "Regression_sin_phi": 0.0026856535114347935, "Regression_cos_phi": 0.0025094610173255205, "Regression_energy": 0.44615626335144043, "HitGrouping": 0.30314671993255615, "Auxiliary": 0.4882153272628784, "Total": 2.493729591369629, "Total_unweighted": 1.5292274642270058}, "train_group": {}, "valid": {"Classification_binary": 0.36847492059071857, "Cardinality": 0.23012501001358032, "Classification": 0.2681616544723511, "Regression_pt": 0.444292147954305, "Regression_eta": 0.00512281929453214, "Regression_sin_phi": 0.0027476074174046516, "Regression_cos_phi": 0.0027338837583859763, "Regression_energy": 0.4370948870976766, "HitGrouping": 0.30192724863688153, "Auxiliary": 0.4940250714619954, "Total": 2.581606864929199, "Total_unweighted": 1.528627920585374, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7883369326591492, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.10030098259449005, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.25540757179260254, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.09708384424448013, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.07036169618368149, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5333899855613708, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7923024296760559, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.09270310401916504, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.1938326060771942, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.0901881530880928, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.059815675020217896, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4947475492954254, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7678524255752563, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.10102058947086334, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.24186551570892334, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.09784125536680222, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.08126089721918106, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5796921849250793, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7766373157501221, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.10004297643899918, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.24191489815711975, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.09691107273101807, "metrics/hit_grouping/calorimeter/all/object_purity": 0.07583006471395493, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5571732521057129, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025713328272104263, "metrics/hit_grouping/calorimeter/background_recall": 0.6245859265327454, "metrics/hit_grouping/calorimeter/completeness": 0.20493124425411224, "metrics/hit_grouping/calorimeter/energy_completeness": 0.24622181057929993, "metrics/hit_grouping/calorimeter/energy_purity": 0.24343931674957275, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.621553897857666, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7822204828262329, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.09214019030332565, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.19259481132030487, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.08962643146514893, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.08539190143346786, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5388474464416504, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7751967906951904, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.09991419315338135, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2555798292160034, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.09651482105255127, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.06941641122102737, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5656234622001648, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7663204669952393, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.11690481007099152, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3320692777633667, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.11274320632219315, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.062274400144815445, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.587157130241394, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.757139265537262, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.11324261873960495, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31423231959342957, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.10861014574766159, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.08450204133987427, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6050451397895813, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.808006763458252, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.33160850405693054, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6028257608413696, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3287692070007324, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.21331200003623962, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4918890595436096, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7935847043991089, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.056220702826976776, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1179858073592186, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.05466455593705177, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.04891452565789223, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5140618681907654, "metrics/hit_grouping/calorimeter/purity": 0.20189718902111053, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8890194892883301, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.03444712236523628, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10461538285017014, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.03330036625266075, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0036409583408385515, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05230769142508507, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8015282154083252, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.03732239454984665, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0610034205019474, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.03641996160149574, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.025530071929097176, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3990877866744995, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7728102207183838, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.1050485223531723, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2552465796470642, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.1017460748553276, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.08026176691055298, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5770264863967896, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.68343585729599, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4031679630279541, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.2591650187969208, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2115260660648346, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6885833740234375, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7750136852264404, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.33875492215156555, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.29047903418540955, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.20696578919887543, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48401570320129395, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6239652037620544, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.39072221517562866, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2295011430978775, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.18942292034626007, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7392297387123108, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6554831266403198, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.390279620885849, "metrics/hit_grouping/tracker/all/object_completeness": 0.2440124899148941, "metrics/hit_grouping/tracker/all/object_purity": 0.1978866010904312, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7017287015914917, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006929850671440363, "metrics/hit_grouping/tracker/background_recall": 0.023988626897335052, "metrics/hit_grouping/tracker/completeness": 0.25759708881378174, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6559173464775085, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6434980034828186, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.414122998714447, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.2907833158969879, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.25089380145072937, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7285649180412292, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6551504135131836, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.3763563632965088, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.21666176617145538, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.1697603017091751, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7058823704719543, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6824418902397156, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3521440029144287, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1692477911710739, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.11010213941335678, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6377426981925964, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6847752332687378, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41265276074409485, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.257346510887146, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.20159754157066345, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.667954683303833, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7433454394340515, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.3968042731285095, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.26345446705818176, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.19637702405452728, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5499334335327148, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6018736362457275, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.35998594760894775, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.22322313487529755, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.19331763684749603, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7724186182022095, "metrics/hit_grouping/tracker/purity": 0.24603714048862457, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8890870213508606, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.008253692649304867, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.009754364378750324, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.007525536231696606, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07428323477506638, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6735632419586182, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.12752722203731537, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.08100143074989319, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.05166073888540268, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5847589373588562, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6216815710067749, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4721090793609619, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.29440566897392273, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.24040362238883972, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8003069758415222, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.19784113764762878, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5750000476837158, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5882524847984314, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.650252640247345, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5664287805557251, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.28788620233535767, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.03562401235103607, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.320106506347656, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.17576764523983002, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.04532782733440399, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5123071074485779, "metrics/particle/class_1/pid_accuracy": 0.8817395567893982, "metrics/particle/class_2/efficiency": 0.3119811415672302, "metrics/particle/class_2/pid_accuracy": 0.07587198913097382, "metrics/particle/class_3/efficiency": 0.44253748655319214, "metrics/particle/class_3/pid_accuracy": 0.8810355067253113, "metrics/particle/class_4/efficiency": 0.47834455966949463, "metrics/particle/class_4/pid_accuracy": 0.11468812823295593, "metrics/particle/class_5/efficiency": 0.4359999895095825, "metrics/particle/class_5/pid_accuracy": 0.08256880939006805, "metrics/particle/count/bias_mean": -6.662760257720947, "metrics/particle/count/mae": 8.403645515441895, "metrics/particle/count/prediction_mean": 63.39323043823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1413879245519638, "metrics/particle/event/energy_response_mean": 0.9494408965110779, "metrics/particle/event/met_abs_error": 14.435848236083984, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1251518875360489, "metrics/particle/event/scalar_pt_response_mean": 0.99658203125, "metrics/particle/event/vector_pt_closure": 0.1133415624499321, "metrics/particle/matched/delta_eta_abs_mean": 0.021989010274410248, "metrics/particle/matched/delta_phi_abs_mean": 0.02270815148949623, "metrics/particle/matched/delta_r_mean": 0.035249654203653336, "metrics/particle/matched/energy_relative_abs_mean": 0.20708796381950378, "metrics/particle/matched/pid_accuracy": 0.8255358338356018, "metrics/particle/matched/pt_relative_abs_mean": 0.20533932745456696, "metrics/particle/matching/duplicate_fraction": 0.10808034986257553, "metrics/particle/matching/efficiency": 0.46480679512023926, "metrics/particle/matching/f1": 0.488013356924057, "metrics/particle/matching/purity": 0.5136589407920837, "metrics/particle/matching/target_coverage_dr0p05": 0.4021894633769989, "metrics/particle/matching/target_coverage_dr0p10": 0.5637511014938354, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.28103452920913696, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.546911895275116, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03479243814945221, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20702418684959412, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20753856003284454, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02359660714864731, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24920324981212616, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01927860639989376, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3928996324539185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03331826627254486, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.0415263175964355, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4824.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.7425661087036133, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.6260695457458496, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.5189905166625977, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.2902534008026123, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3903910517692566, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 20.717546463012695, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03867284208536148, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.212768077850342, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0833115205168724, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.895627975463867, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9283.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4411119222640991, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.4148406982421875, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 17.3735294342041, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.079100608825684, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6734708547592163, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.0417453050613403, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.705418586730957, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.71079421043396, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 23.316862106323242, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.033773306757211685, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20742227137088776, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.20980195701122284, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.03517807647585869, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.25478512048721313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.05430711433291435, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.613321542739868, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.2056010663509369, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.806123733520508, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1068.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.0750718116760254, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.1184022426605225, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.921499729156494, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.697700500488281, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.41824042797088623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.48272705078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08705665916204453, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.8282883167266846, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11784430593252182, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.493273735046387, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2171.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.353525310754776, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -13.837431907653809, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 16.719247817993164, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.366166353225708, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.40686073899269104, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.13324834406375885, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 16.4581241607666, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.7326080799102783, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 35.02261734008789, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03596283867955208, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20671668648719788, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20392687618732452, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.018423309549689293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24517427384853363, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0016979529755190015, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.47080832719802856, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.022085217759013176, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.167268991470337, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 10601.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6307247877120972, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.5140750408172607, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.0001368522644043, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.783856153488159, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3871367573738098, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.04607105255127, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.01428146380931139, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.1177592277526855, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02976858615875244, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1391823291778564, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20166.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.49451881647109985, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.496835231781006, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.613597869873047, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.467082977294922, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.4068024158477783, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.7965245246887207, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.718520164489746, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.6330206394195557, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 14.512063980102539, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03547872602939606, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20685233175754547, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.2053637057542801, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.021021386608481407, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24699021875858307, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.01024677138775587, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8792465925216675, "metrics/particle/momentum/all/accepted/pz_bias": 0.03725426644086838, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.400935173034668, "metrics/particle/momentum/all/accepted/samples": 16493.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6940675973892212, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.157548666000366, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.3530399799346924, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.1325862407684326, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.39022770524024963, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.65019989013672, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.026438962668180466, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.62533438205719, "metrics/particle/momentum/all/assigned/pz_bias": 0.05153492093086243, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.41687536239624, "metrics/particle/momentum/all/assigned/samples": 31620.0, "metrics/particle/momentum/all/efficiency": 0.46598294377326965, "metrics/particle/momentum/all/event/energy_bias": -10.755437850952148, "metrics/particle/momentum/all/event/p3_error_mean": 40.49247741699219, "metrics/particle/momentum/all/event/pt_vector_error_mean": 20.008167266845703, "metrics/particle/momentum/all/event/px_bias": -3.487133502960205, "metrics/particle/momentum/all/event/py_bias": -3.971518039703369, "metrics/particle/momentum/all/event/pz_abs_mean": 29.92095184326172, "metrics/particle/momentum/all/event/pz_bias": -1.8103818893432617, "metrics/particle/momentum/all/event/pz_residual_rms": 43.10661697387695, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04438524693250656, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2262992113828659, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22408586740493774, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.04890776053071022, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.26366546750068665, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.011975351721048355, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.0219866037368774, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.014973447658121586, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.46683669090271, "metrics/particle/momentum/class_1/accepted/samples": 8601.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5101661086082458, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.7301634550094604, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.9976106882095337, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.7305426597595215, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3397987484931946, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 13.343746185302734, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.024625975638628006, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.5773911476135254, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.013526546768844128, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.8628883361816406, "metrics/particle/momentum/class_1/assigned/samples": 15106.0, "metrics/particle/momentum/class_1/efficiency": 0.5154311656951904, "metrics/particle/momentum/class_1/event/energy_bias": 16.355520248413086, "metrics/particle/momentum/class_1/event/p3_error_mean": 32.790889739990234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 19.185144424438477, "metrics/particle/momentum/class_1/event/px_bias": -0.06854818761348724, "metrics/particle/momentum/class_1/event/py_bias": -2.9912569522857666, "metrics/particle/momentum/class_1/event/pz_abs_mean": 22.649816513061523, "metrics/particle/momentum/class_1/event/pz_bias": -4.825918197631836, "metrics/particle/momentum/class_1/event/pz_residual_rms": 32.10832214355469, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.044733572751283646, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23731760680675507, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23333275318145752, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07563403993844986, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27466318011283875, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02957283705472946, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7888203859329224, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.12429478764533997, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8010568618774414, "metrics/particle/momentum/class_2/accepted/samples": 913.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9383028745651245, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7107325792312622, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9677587747573853, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.515028178691864, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38776493072509766, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.415170907974243, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06454720348119736, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.084242820739746, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.35097599029541016, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.133889675140381, "metrics/particle/momentum/class_2/assigned/samples": 2076.0, "metrics/particle/momentum/class_2/efficiency": 0.30668458342552185, "metrics/particle/momentum/class_2/event/energy_bias": -37.79047775268555, "metrics/particle/momentum/class_2/event/p3_error_mean": 31.45590591430664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.033514976501465, "metrics/particle/momentum/class_2/event/px_bias": -0.3496239483356476, "metrics/particle/momentum/class_2/event/py_bias": -1.3881282806396484, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.741302490234375, "metrics/particle/momentum/class_2/event/pz_bias": -1.7495485544204712, "metrics/particle/momentum/class_2/event/pz_residual_rms": 36.79503631591797, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02259746752679348, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1772267371416092, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17676322162151337, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024778973311185837, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21855910122394562, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.004019053187221289, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.5211305618286133, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.04435772821307182, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.897121548652649, "metrics/particle/momentum/class_3/accepted/samples": 6718.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8595386147499084, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.055481433868408, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.167664527893066, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.035577774047852, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.446105420589447, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.452741622924805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.021529218181967735, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.4273886680603027, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07228361815214157, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.389338493347168, "metrics/particle/momentum/class_3/assigned/samples": 13981.0, "metrics/particle/momentum/class_3/efficiency": 0.4417702257633209, "metrics/particle/momentum/class_3/event/energy_bias": 21.81494140625, "metrics/particle/momentum/class_3/event/p3_error_mean": 26.71086311340332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 15.01972770690918, "metrics/particle/momentum/class_3/event/px_bias": -2.608931064605713, "metrics/particle/momentum/class_3/event/py_bias": -1.3124171495437622, "metrics/particle/momentum/class_3/event/pz_abs_mean": 18.862817764282227, "metrics/particle/momentum/class_3/event/pz_bias": 5.351476669311523, "metrics/particle/momentum/class_3/event/pz_residual_rms": 29.145160675048828, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.042258016765117645, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.21410898864269257, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.22066769003868103, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0846962183713913, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2591978907585144, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.05376344174146652, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.1835670471191406, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.267754465341568, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 4.286838531494141, "metrics/particle/momentum/class_4/accepted/samples": 186.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6748996376991272, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.449642181396484, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.5643463134765625, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.3185906410217285, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.375, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.220890045166016, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0416666679084301, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.1065709590911865, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2472177892923355, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.4430832862854, "metrics/particle/momentum/class_4/assigned/samples": 336.0, "metrics/particle/momentum/class_4/efficiency": 0.5109890103340149, "metrics/particle/momentum/class_4/event/energy_bias": -5.4670796394348145, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.460659503936768, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.06155252456665, "metrics/particle/momentum/class_4/event/px_bias": 0.2529328763484955, "metrics/particle/momentum/class_4/event/py_bias": 0.9950428009033203, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.9023733139038086, "metrics/particle/momentum/class_4/event/pz_bias": -0.10650427639484406, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.976308822631836, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03842107951641083, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24148216843605042, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2417140156030655, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.1027711033821106, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2810901701450348, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02666666731238365, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.280212879180908, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.3249243199825287, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.094400405883789, "metrics/particle/momentum/class_5/accepted/samples": 75.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.3963446617126465, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4511427879333496, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4579803943634033, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.09799715876579285, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3140496015548706, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6881412267684937, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.12396693974733353, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.115569114685059, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.09568241983652115, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.004142761230469, "metrics/particle/momentum/class_5/assigned/samples": 121.0, "metrics/particle/momentum/class_5/efficiency": 0.4716981053352356, "metrics/particle/momentum/class_5/event/energy_bias": -5.668346405029297, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.241645812988281, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.132990837097168, "metrics/particle/momentum/class_5/event/px_bias": -0.7129635810852051, "metrics/particle/momentum/class_5/event/py_bias": 0.7252411842346191, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.6346538066864014, "metrics/particle/momentum/class_5/event/pz_bias": -0.479887992143631, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.121149063110352, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6910575032234192, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6968439817428589, "metrics/set_slot/angular_recall_all_dr0p10": 0.8234949707984924, "metrics/set_slot/angular_recall_selected_dr0p10": 0.594325602054596, "metrics/set_slot/soft_count_bias_mean": -0.15992076694965363}, "step_train_time": 4.273864984512329, "step_valid_time": 9697.887388467789, "step_total_time": 9702.16125369072, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.065204631028159, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.32557249953591494, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2868, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2441, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2232, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.806408985794516, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.851115760111576, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8281594571670907, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.148884239888424, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7373637264618434, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7782426778242678, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7572519083969466, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.806408985794516, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0324797939456578, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2913847298831287, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2181, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1843, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1757, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8147656940760389, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8450252177900046, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.829619626378573, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15497478220999542, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7767462422634837, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8055937643282898, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7909070447895566, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8147656940760389, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999309766690039, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2937886784668562, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1229, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1018, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 979, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7903726708074534, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8283157038242474, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8088994835121176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17168429617575265, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7600931677018633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7965825874694874, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7779102105681367, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7903726708074534} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_40000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..5afd6ae803bb29d61b6403051919c247b344f8e9 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.895016107636915, "train": {"Classification_binary": 0.20928284525871277, "Cardinality": 0.13597911596298218, "Classification": 0.08416189253330231, "Regression_pt": 0.16748285293579102, "Regression_eta": 0.0017309194663539529, "Regression_sin_phi": 0.0009038360440172255, "Regression_cos_phi": 0.0010004653595387936, "Regression_energy": 0.16768129169940948, "HitGrouping": 0.22893083095550537, "Auxiliary": 0.27541813254356384, "Total": 1.2692735195159912, "Total_unweighted": 0.6322441032971255}, "train_group": {}, "valid": {"Classification_binary": 0.1845386028289795, "Cardinality": 0.13565772771835327, "Classification": 0.09283845623334248, "Regression_pt": 0.16706637541453043, "Regression_eta": 0.0016893854675193627, "Regression_sin_phi": 0.0009538172744214535, "Regression_cos_phi": 0.0009695499514540037, "Regression_energy": 0.1662184198697408, "HitGrouping": 0.22729980945587158, "Auxiliary": 0.26678844292958576, "Total": 1.2381761868794758, "Total_unweighted": 0.614274607039988, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7390709519386292, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3717312216758728, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43987908959388733, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3617285192012787, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36948639154434204, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5915745496749878, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6845735311508179, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.31986019015312195, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.420196533203125, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3092457354068756, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.34500351548194885, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6668925881385803, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7577136158943176, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36372631788253784, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.41085630655288696, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35631972551345825, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.35713285207748413, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5549013614654541, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7451542019844055, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3623676002025604, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.42103344202041626, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35383787751197815, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3600201904773712, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.576747715473175, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025888388976454735, "metrics/hit_grouping/calorimeter/background_recall": 0.632445216178894, "metrics/hit_grouping/calorimeter/completeness": 0.4173393249511719, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4433841109275818, "metrics/hit_grouping/calorimeter/energy_purity": 0.4382208287715912, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4874745309352875, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7634355425834656, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3781864047050476, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3484025001525879, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3720012605190277, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.38094431161880493, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5292923450469971, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7403354048728943, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.36939433217048645, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4614264965057373, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.35945653915405273, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3700242042541504, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5980106592178345, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7114801406860352, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3215619623661041, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5326760411262512, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.30953678488731384, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.30530551075935364, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6550372838973999, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7302033305168152, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.38438427448272705, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5066957473754883, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3752380609512329, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.38659828901290894, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6161320209503174, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7448864579200745, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4765322804450989, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6352694630622864, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.46458399295806885, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4451393485069275, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5735217332839966, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7612615823745728, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32408949732780457, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.3015267252922058, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3166598677635193, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3205549120903015, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5348199009895325, "metrics/hit_grouping/calorimeter/purity": 0.41123440861701965, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8317661881446838, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07171454280614853, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15076923370361328, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07510684430599213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.037070490419864655, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05384615436196327, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7537045478820801, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14504209160804749, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11687571555376053, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14405545592308044, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14464326202869415, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.41448119282722473, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7428162693977356, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3810622990131378, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4442877769470215, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.371844619512558, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37929120659828186, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5972258448600769, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7133072018623352, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43693962693214417, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4451526999473572, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4611741900444031, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6189479827880859, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7772153615951538, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.335950642824173, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4776758551597595, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4831404387950897, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4772854745388031, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6621453762054443, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44908246397972107, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38906025886535645, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40740641951560974, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6986689567565918, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.688007116317749, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43571698665618896, "metrics/hit_grouping/tracker/all/object_completeness": 0.4142525792121887, "metrics/hit_grouping/tracker/all/object_purity": 0.43077918887138367, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6548271179199219, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0044501107186079025, "metrics/hit_grouping/tracker/background_recall": 0.023514777421951294, "metrics/hit_grouping/tracker/completeness": 0.43465691804885864, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4988357424736023, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6978090405464172, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4174942970275879, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4653208255767822, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.48824813961982727, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6470159292221069, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6755150556564331, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4551684856414795, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.39292454719543457, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.41354605555534363, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.673710286617279, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6795209646224976, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45550161600112915, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3235490620136261, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3215161859989166, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6521035432815552, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7117077708244324, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45857298374176025, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43751585483551025, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4472223222255707, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6232027411460876, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7464644312858582, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4833555221557617, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4606654942035675, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4436608850955963, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5645805597305298, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.646852433681488, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39756497740745544, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3757854700088501, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40709298849105835, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.711894154548645, "metrics/hit_grouping/tracker/purity": 0.41411712765693665, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8856326341629028, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01824500411748886, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03376663476228714, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04017196595668793, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08166811615228653, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6740120649337769, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21669258177280426, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2284950464963913, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.23842686414718628, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5837221145629883, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6627466678619385, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.517383873462677, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48710569739341736, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5057585835456848, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7408063411712646, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4316942095756531, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5621642470359802, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6201850175857544, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5938101410865784, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05403313785791397, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06300542503595352, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.95285415649414, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2821117043495178, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1516718715429306, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8569251894950867, "metrics/particle/class_1/pid_accuracy": 0.9633563756942749, "metrics/particle/class_2/efficiency": 0.40442001819610596, "metrics/particle/class_2/pid_accuracy": 0.6219140291213989, "metrics/particle/class_3/efficiency": 0.7032158374786377, "metrics/particle/class_3/pid_accuracy": 0.9637572765350342, "metrics/particle/class_4/efficiency": 0.707410991191864, "metrics/particle/class_4/pid_accuracy": 0.35374149680137634, "metrics/particle/class_5/efficiency": 0.8799999952316284, "metrics/particle/class_5/pid_accuracy": 0.40454545617103577, "metrics/particle/count/bias_mean": -3.9212238788604736, "metrics/particle/count/mae": 5.018880367279053, "metrics/particle/count/prediction_mean": 66.134765625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07679944485425949, "metrics/particle/event/energy_response_mean": 0.9765302538871765, "metrics/particle/event/met_abs_error": 8.086280822753906, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06316735595464706, "metrics/particle/event/scalar_pt_response_mean": 0.9899029731750488, "metrics/particle/event/vector_pt_closure": 0.06454356759786606, "metrics/particle/matched/delta_eta_abs_mean": 0.014583691954612732, "metrics/particle/matched/delta_phi_abs_mean": 0.014633090235292912, "metrics/particle/matched/delta_r_mean": 0.02314600721001625, "metrics/particle/matched/energy_relative_abs_mean": 0.15831251442432404, "metrics/particle/matched/pid_accuracy": 0.9397165775299072, "metrics/particle/matched/pt_relative_abs_mean": 0.15868744254112244, "metrics/particle/matching/duplicate_fraction": 0.06317001581192017, "metrics/particle/matching/efficiency": 0.7515194416046143, "metrics/particle/matching/f1": 0.7731572985649109, "metrics/particle/matching/purity": 0.796078085899353, "metrics/particle/matching/target_coverage_dr0p05": 0.7105087041854858, "metrics/particle/matching/target_coverage_dr0p10": 0.8124082088470459, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5305466055870056, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7896400094032288, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022382128983736038, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15429896116256714, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15487506985664368, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012685619294643402, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19559980928897858, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00926955882459879, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8024615049362183, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.020208382979035378, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.135242462158203, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8091.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2650952637195587, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2544394731521606, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6103898286819458, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.441866397857666, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15290947258472443, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 30.72098731994629, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019862262532114983, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2388192415237427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0744321420788765, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.596461296081543, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10019.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7398500442504883, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9051263928413391, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.050922393798828, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.678045272827148, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.18180203437805176, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.02948358654975891, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.555768013000488, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.43896323442459106, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.532882690429688, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02311604656279087, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15938344597816467, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15969303250312805, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01888160966336727, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1993764042854309, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029842648655176163, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7058459520339966, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06942101567983627, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.342380523681641, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1843.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.397567480802536, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5735893249511719, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.253345251083374, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.090158224105835, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1871679574251175, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 68.35946655273438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06742950528860092, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.717088460922241, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.05831955373287201, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.753076076507568, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2447.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6100628972053528, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.923415422439575, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.566340446472168, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.493683099746704, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08146756887435913, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15172864496707916, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.392984390258789, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.8989505767822266, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.596311569213867, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02339451014995575, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15979646146297455, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1601029336452484, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015470360405743122, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20054474472999573, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006598284235224128, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26948198676109314, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.01745194010436535, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8029515743255615, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16671.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23967310786247253, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5312302112579346, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.125840425491333, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.945192277431488, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15210120379924774, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.00977897644043, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0059619019739329815, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5263561010360718, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006897763814777136, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3599791526794434, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20631.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.77767413854599, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.8759872317314148, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.194416999816895, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.669820785522461, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.05115003511309624, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.3534693717956543, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.399353504180908, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.47092416882514954, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.191373825073242, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02306734025478363, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15809595584869385, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15848465263843536, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01485978253185749, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19897259771823883, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005299755837768316, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5310702323913574, "metrics/particle/momentum/all/accepted/pz_bias": -0.01227229367941618, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7597624063491821, "metrics/particle/momentum/all/accepted/samples": 26605.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2590425908565521, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4505729675292969, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4298168420791626, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.2541298866271973, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15493851900100708, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 26.337236404418945, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014714324846863747, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9040003418922424, "metrics/particle/momentum/all/assigned/pz_bias": 0.022543910890817642, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2800822257995605, "metrics/particle/momentum/all/assigned/samples": 33097.0, "metrics/particle/momentum/all/efficiency": 0.7516810894012451, "metrics/particle/momentum/all/event/energy_bias": -5.70452880859375, "metrics/particle/momentum/all/event/p3_error_mean": 21.22017478942871, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.886104583740234, "metrics/particle/momentum/all/event/px_bias": -0.31441962718963623, "metrics/particle/momentum/all/event/py_bias": 1.2312241792678833, "metrics/particle/momentum/all/event/pz_abs_mean": 15.41728687286377, "metrics/particle/momentum/all/event/pz_bias": 0.8669896125793457, "metrics/particle/momentum/all/event/pz_residual_rms": 24.841777801513672, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02458968758583069, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1491510421037674, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15021266043186188, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.023467032238841057, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19009579718112946, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0058700209483504295, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5652331709861755, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.042373690754175186, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7594164609909058, "metrics/particle/momentum/class_1/accepted/samples": 14310.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1262112706899643, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.38134509325027466, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4250588119029999, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.27800363302230835, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0913568064570427, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.548732280731201, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009665107354521751, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7287068963050842, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.022462397813796997, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.483287811279297, "metrics/particle/momentum/class_1/assigned/samples": 16244.0, "metrics/particle/momentum/class_1/efficiency": 0.8575537800788879, "metrics/particle/momentum/class_1/event/energy_bias": 5.2017388343811035, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.730690002441406, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.929289817810059, "metrics/particle/momentum/class_1/event/px_bias": 0.20860949158668518, "metrics/particle/momentum/class_1/event/py_bias": 0.8172682523727417, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.420476913452148, "metrics/particle/momentum/class_1/event/pz_bias": -0.8960137963294983, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.878128051757812, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03276345506310463, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20146197080612183, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2027270495891571, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.01959453709423542, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24204140901565552, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02618243172764778, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7724891901016235, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.06490666419267654, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.153619766235352, "metrics/particle/momentum/class_2/accepted/samples": 1184.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.707845151424408, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8907020688056946, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0075703859329224, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6137863397598267, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.32531824707984924, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.8376851081848145, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06506364792585373, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.946556806564331, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08509135246276855, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.173210144042969, "metrics/particle/momentum/class_2/assigned/samples": 2121.0, "metrics/particle/momentum/class_2/efficiency": 0.39771580696105957, "metrics/particle/momentum/class_2/event/energy_bias": -11.211358070373535, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.07362174987793, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.75755500793457, "metrics/particle/momentum/class_2/event/px_bias": 0.580415666103363, "metrics/particle/momentum/class_2/event/py_bias": -0.5972389578819275, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.058061599731445, "metrics/particle/momentum/class_2/event/pz_bias": 1.9264328479766846, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.880441665649414, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019945740699768066, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.164597749710083, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16427412629127502, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.006380531471222639, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2048509418964386, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016819286393001676, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.31727081537246704, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.016797540709376335, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1354094743728638, "metrics/particle/momentum/class_3/accepted/samples": 10702.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3455718755722046, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.755988597869873, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.6473653316497803, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.4702908992767334, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20198117196559906, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 39.770103454589844, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012294505722820759, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7730416059494019, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06329169124364853, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.115691661834717, "metrics/particle/momentum/class_3/assigned/samples": 14234.0, "metrics/particle/momentum/class_3/efficiency": 0.7037548422813416, "metrics/particle/momentum/class_3/event/energy_bias": 3.229259490966797, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.416328430175781, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.030104637145996, "metrics/particle/momentum/class_3/event/px_bias": -0.9994646310806274, "metrics/particle/momentum/class_3/event/py_bias": 0.529411792755127, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.779741287231445, "metrics/particle/momentum/class_3/event/pz_bias": -0.7374306917190552, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.872623443603516, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026912691071629524, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17454801499843597, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16707274317741394, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0702831819653511, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2045467495918274, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.014925372786819935, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0291998386383057, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.27377036213874817, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.5533840656280518, "metrics/particle/momentum/class_4/accepted/samples": 268.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2874237596988678, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8658219575881958, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.615735411643982, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5037599802017212, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.18529412150382996, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.268953323364258, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01764705963432789, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2408907413482666, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.24698686599731445, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.638033151626587, "metrics/particle/momentum/class_4/assigned/samples": 340.0, "metrics/particle/momentum/class_4/efficiency": 0.7362637519836426, "metrics/particle/momentum/class_4/event/energy_bias": -1.4756419658660889, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.099562168121338, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1647026538848877, "metrics/particle/momentum/class_4/event/px_bias": -0.045393623411655426, "metrics/particle/momentum/class_4/event/py_bias": 0.12582309544086456, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8678529262542725, "metrics/particle/momentum/class_4/event/pz_bias": 0.5200613141059875, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.800345420837402, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016768313944339752, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17700046300888062, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17074695229530334, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.031128598377108574, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21484068036079407, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.028368793427944183, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9202302694320679, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3554900884628296, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.8787286281585693, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03432447835803032, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.39708852767944336, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.310038685798645, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10639213770627975, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.10126582533121109, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.904569685459137, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06962025165557861, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5794804096221924, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.34388047456741333, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.096195697784424, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.8867924809455872, "metrics/particle/momentum/class_5/event/energy_bias": -1.4485268592834473, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.676262378692627, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.017091989517212, "metrics/particle/momentum/class_5/event/px_bias": -0.05858658254146576, "metrics/particle/momentum/class_5/event/py_bias": 0.35596010088920593, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.414368987083435, "metrics/particle/momentum/class_5/event/pz_bias": 0.053939953446388245, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.4279279708862305, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4025847315788269, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4042886793613434, "metrics/set_slot/angular_recall_all_dr0p10": 0.9446127414703369, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8039979338645935, "metrics/set_slot/soft_count_bias_mean": -0.8585536479949951}, "step_train_time": 4.606922388076782, "step_valid_time": 12154.442369937897, "step_total_time": 12159.04929280281, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9932379923720777, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1677200301109386, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2857, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2829, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9438387842748596, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9482243611018918, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9460264900662252, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0517756388981082, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9345887016848364, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9389312977099237, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9367549668874172, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9438387842748596, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9928960005104696, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15625205878879755, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2117, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2106, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9358974358974359, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9455113890129522, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9406798489224617, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05448861098704779, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9310344827586207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9405984814649397, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9357920462119529, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9358974358974359, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9924621594814734, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16080837495984923, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1267, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9153726708074534, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.930544593528019, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9228962818003914, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06945540647198106, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9107142857142857, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.925808997632202, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9181996086105675, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9153726708074534} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_41000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..478dda52100ded62d1910d3720d3b2e5646e2dc2 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9423915103278377, "train": {"Classification_binary": 0.21244804561138153, "Cardinality": 0.1394532471895218, "Classification": 0.08646313846111298, "Regression_pt": 0.18503645062446594, "Regression_eta": 0.0015942059690132737, "Regression_sin_phi": 0.0009092633263207972, "Regression_cos_phi": 0.0008970100898295641, "Regression_energy": 0.18324793875217438, "HitGrouping": 0.22742484509944916, "Auxiliary": 0.2844374179840088, "Total": 1.288887858390808, "Total_unweighted": 0.6705960528342985}, "train_group": {}, "valid": {"Classification_binary": 0.19047053654988608, "Cardinality": 0.1367601752281189, "Classification": 0.09124916791915894, "Regression_pt": 0.17833868662516275, "Regression_eta": 0.0017734930540124576, "Regression_sin_phi": 0.000967009924352169, "Regression_cos_phi": 0.0009938972846915324, "Regression_energy": 0.17743057012557983, "HitGrouping": 0.2315922180811564, "Auxiliary": 0.27327434221903485, "Total": 1.2718657652537029, "Total_unweighted": 0.6412233614828438, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7382634878158569, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.35681572556495667, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43553414940834045, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.34869658946990967, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.35255464911460876, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6017757654190063, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6828747391700745, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.30338072776794434, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4161301255226135, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.29184019565582275, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.32730308175086975, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6746865510940552, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7547340393066406, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3536245822906494, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4095134735107422, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3455258905887604, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34291237592697144, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5659539103507996, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7429884076118469, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.35014477372169495, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41848024725914, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.34173083305358887, "metrics/hit_grouping/calorimeter/all/object_purity": 0.34461510181427, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5872340202331543, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025654643774032593, "metrics/hit_grouping/calorimeter/background_recall": 0.6298788785934448, "metrics/hit_grouping/calorimeter/completeness": 0.40501558780670166, "metrics/hit_grouping/calorimeter/energy_completeness": 0.433120995759964, "metrics/hit_grouping/calorimeter/energy_purity": 0.4280247390270233, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.49366647005081177, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7635117769241333, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.36539652943611145, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34655120968818665, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.35920774936676025, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3654024600982666, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5375933051109314, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7378088235855103, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3593083918094635, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4554827809333801, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.34844598174095154, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.352700799703598, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6073508262634277, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7049448490142822, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.30831262469291687, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5321704149246216, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.29774010181427, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29218900203704834, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6713436841964722, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7272670269012451, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3773937225341797, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5108689069747925, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.36806076765060425, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.370635449886322, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6196200847625732, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7337760925292969, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.46299290657043457, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6368393301963806, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.44884398579597473, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.43647006154060364, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6117216348648071, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7610687613487244, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3064100742340088, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2912146747112274, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.29971790313720703, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3048875033855438, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.549283504486084, "metrics/hit_grouping/calorimeter/purity": 0.39899522066116333, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8434420228004456, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07087454944849014, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1599999964237213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07218559086322784, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03355854004621506, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05538461357355118, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7499232292175293, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.13511617481708527, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.108323834836483, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13283826410770416, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13200128078460693, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.446408212184906, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7404484748840332, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.36846479773521423, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4418284296989441, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.35949060320854187, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.36347368359565735, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6066697239875793, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7150282263755798, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.443365216255188, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.43542757630348206, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.44727814197540283, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6267184615135193, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7650690078735352, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32809871435165405, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4657239019870758, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48279017210006714, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48850253224372864, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6677634716033936, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4454595446586609, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3838411271572113, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3943987786769867, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6874852180480957, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6908974051475525, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4349140226840973, "metrics/hit_grouping/tracker/all/object_completeness": 0.4070422351360321, "metrics/hit_grouping/tracker/all/object_purity": 0.41855666041374207, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.651520848274231, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0056854053400456905, "metrics/hit_grouping/tracker/background_recall": 0.031629450619220734, "metrics/hit_grouping/tracker/completeness": 0.4256368577480316, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5171231031417847, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7013166546821594, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42651480436325073, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46155720949172974, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4750133156776428, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6437357664108276, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6828183531761169, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45059967041015625, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38613006472587585, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3993518650531769, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6620978713035583, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6763520240783691, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43689319491386414, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3082456886768341, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3136356770992279, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6575646996498108, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7139368057250977, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45866283774375916, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42713814973831177, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.429971843957901, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6155643463134766, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7379709482192993, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45938748121261597, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4394315481185913, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43942710757255554, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5758988261222839, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6526058316230774, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3996722102165222, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37516719102859497, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40001583099365234, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7116600275039673, "metrics/hit_grouping/tracker/purity": 0.4060271084308624, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8850041031837463, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01607297919690609, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02784041315317154, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03335728496313095, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08774977922439575, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6655752062797546, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21358217298984528, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2131495475769043, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22824490070343018, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5987558364868164, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6674048900604248, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5170297026634216, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48064717650413513, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.49256882071495056, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7341361045837402, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4314361810684204, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5736657381057739, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6374062299728394, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5905364155769348, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05429117754101753, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.059731606394052505, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.20102310180664, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29962417483329773, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1691843569278717, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.838326096534729, "metrics/particle/class_1/pid_accuracy": 0.961103618144989, "metrics/particle/class_2/efficiency": 0.40139105916023254, "metrics/particle/class_2/pid_accuracy": 0.5810508728027344, "metrics/particle/class_3/efficiency": 0.6951494216918945, "metrics/particle/class_3/pid_accuracy": 0.9586458206176758, "metrics/particle/class_4/efficiency": 0.6756496429443359, "metrics/particle/class_4/pid_accuracy": 0.35185185074806213, "metrics/particle/class_5/efficiency": 0.8420000076293945, "metrics/particle/class_5/pid_accuracy": 0.3919239938259125, "metrics/particle/count/bias_mean": -3.6686198711395264, "metrics/particle/count/mae": 4.925130367279053, "metrics/particle/count/prediction_mean": 66.38736724853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07689335942268372, "metrics/particle/event/energy_response_mean": 0.97198486328125, "metrics/particle/event/met_abs_error": 7.958444595336914, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06151897832751274, "metrics/particle/event/scalar_pt_response_mean": 0.9954985976219177, "metrics/particle/event/vector_pt_closure": 0.06495460122823715, "metrics/particle/matched/delta_eta_abs_mean": 0.015051223337650299, "metrics/particle/matched/delta_phi_abs_mean": 0.014890714548528194, "metrics/particle/matched/delta_r_mean": 0.02372000180184841, "metrics/particle/matched/energy_relative_abs_mean": 0.16453516483306885, "metrics/particle/matched/pid_accuracy": 0.9345941543579102, "metrics/particle/matched/pt_relative_abs_mean": 0.16524292528629303, "metrics/particle/matching/duplicate_fraction": 0.07057888805866241, "metrics/particle/matching/efficiency": 0.7385554909706116, "metrics/particle/matching/f1": 0.7584133744239807, "metrics/particle/matching/purity": 0.7793686389923096, "metrics/particle/matching/target_coverage_dr0p05": 0.7012805938720703, "metrics/particle/matching/target_coverage_dr0p10": 0.8081798553466797, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5058918595314026, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7782372832298279, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02280353754758835, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16302523016929626, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16423557698726654, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022429488599300385, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20518162846565247, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01000506617128849, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8693603873252869, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.11267884820699692, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2617692947387695, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7896.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29298603534698486, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4222320318222046, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4668384790420532, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3009649515151978, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16948136687278748, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.799718856811523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020385729148983955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2993181943893433, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.13060787320137024, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.480377674102783, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10007.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7220190167427063, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3960070610046387, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.185184478759766, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.633283615112305, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2685120403766632, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.06290420889854431, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.728620529174805, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.128239154815674, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.272047996520996, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02372540719807148, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16612814366817474, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16430452466011047, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01485359575599432, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20553186535835266, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03579418361186981, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.718225121498108, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.13693644106388092, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.1762566566467285, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1788.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5055893659591675, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.04512095451355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.7516491413116455, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.5788118839263916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2193627506494522, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 75.41059112548828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0674019604921341, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.761904239654541, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2085762619972229, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.780964374542236, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2448.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5918570160865784, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.911005973815918, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.781759262084961, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.374631643295288, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0651697963476181, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09266284108161926, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.62729549407959, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1262685060501099, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.600072860717773, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02414807304739952, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16529898345470428, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16583268344402313, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025024538859725, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20705646276474, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008518924005329609, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2838769853115082, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02156728506088257, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8016892671585083, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16434.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23408876359462738, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.536340594291687, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2376428842544556, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0673482418060303, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16471725702285767, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.835624694824219, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005944901145994663, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5183645486831665, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.016180260106921196, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8435522317886353, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20690.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7666184902191162, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7900477051734924, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.076663970947266, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.480949401855469, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.1268531233072281, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.33266907930374146, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.58749532699585, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.42719024419784546, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.636169910430908, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023712655529379845, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16466833651065826, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16524522006511688, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023543711751699448, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20638716220855713, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006011180114001036, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5590738654136658, "metrics/particle/momentum/all/accepted/pz_bias": 0.05701015889644623, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.773399829864502, "metrics/particle/momentum/all/accepted/samples": 26118.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2719230651855469, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5394665002822876, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.492518424987793, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3233708143234253, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17019158601760864, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 23.441162109375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014843868091702461, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.919848620891571, "metrics/particle/momentum/all/assigned/pz_bias": 0.06493761390447617, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.028982162475586, "metrics/particle/momentum/all/assigned/samples": 33145.0, "metrics/particle/momentum/all/efficiency": 0.7379216551780701, "metrics/particle/momentum/all/event/energy_bias": -7.516964912414551, "metrics/particle/momentum/all/event/p3_error_mean": 21.373903274536133, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.730173110961914, "metrics/particle/momentum/all/event/px_bias": -0.460534930229187, "metrics/particle/momentum/all/event/py_bias": 0.4882362186908722, "metrics/particle/momentum/all/event/pz_abs_mean": 15.809980392456055, "metrics/particle/momentum/all/event/pz_bias": 1.4291609525680542, "metrics/particle/momentum/all/event/pz_residual_rms": 25.54380226135254, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025531766936182976, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16136181354522705, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16264282166957855, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.028697941452264786, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20308852195739746, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006292006466537714, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.612644374370575, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.07830050587654114, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7790725231170654, "metrics/particle/momentum/class_1/accepted/samples": 13986.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13811656832695007, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4137604236602783, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5077781081199646, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.35714781284332275, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10695351660251617, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.20813512802124, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010356306098401546, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7772706747055054, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.1038411557674408, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.281033754348755, "metrics/particle/momentum/class_1/assigned/samples": 16222.0, "metrics/particle/momentum/class_1/efficiency": 0.8381374478340149, "metrics/particle/momentum/class_1/event/energy_bias": 8.984149932861328, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.714828491210938, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.042667388916016, "metrics/particle/momentum/class_1/event/px_bias": -1.3613563776016235, "metrics/particle/momentum/class_1/event/py_bias": 0.0021385252475738525, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.534274101257324, "metrics/particle/momentum/class_1/event/pz_bias": 2.8714652061462402, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.401565551757812, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03312385454773903, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20616589486598969, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20633713901042938, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03823138400912285, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24708744883537292, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030405405908823013, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8704441785812378, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.15251581370830536, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.397696495056152, "metrics/particle/momentum/class_2/accepted/samples": 1184.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7425659894943237, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.811801016330719, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.096597671508789, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7093619704246521, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3293357789516449, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.737889289855957, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.061346862465143204, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8910248279571533, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.14516369998455048, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.9733052253723145, "metrics/particle/momentum/class_2/assigned/samples": 2168.0, "metrics/particle/momentum/class_2/efficiency": 0.39771580696105957, "metrics/particle/momentum/class_2/event/energy_bias": -17.737089157104492, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.596891403198242, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.974555969238281, "metrics/particle/momentum/class_2/event/px_bias": 0.9610010385513306, "metrics/particle/momentum/class_2/event/py_bias": -0.7698150277137756, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.45230484008789, "metrics/particle/momentum/class_2/event/pz_bias": -0.7923074960708618, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.4462833404541, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020215364173054695, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1638878732919693, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1638079583644867, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023776711896061897, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20552000403404236, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0024611889384686947, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.31078535318374634, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01102342177182436, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0570896863937378, "metrics/particle/momentum/class_3/accepted/samples": 10564.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3535827100276947, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.90073823928833, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.6632497310638428, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5061874389648438, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2165462076663971, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 35.21683120727539, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012279840186238289, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.754526674747467, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0004199052636977285, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.724846124649048, "metrics/particle/momentum/class_3/assigned/samples": 14251.0, "metrics/particle/momentum/class_3/efficiency": 0.6946800947189331, "metrics/particle/momentum/class_3/event/energy_bias": 4.113554000854492, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.756641387939453, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.574495315551758, "metrics/particle/momentum/class_3/event/px_bias": 0.08432625234127045, "metrics/particle/momentum/class_3/event/py_bias": 0.7442947030067444, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.310869216918945, "metrics/particle/momentum/class_3/event/pz_bias": -1.4597890377044678, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.475471496582031, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027652665972709656, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17823822796344757, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1710563749074936, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0481179803609848, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21058951318264008, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.015564202331006527, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1805474758148193, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.47992560267448425, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.7108304500579834, "metrics/particle/momentum/class_4/accepted/samples": 257.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.32610654830932617, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.3577752113342285, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4569737911224365, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3141117095947266, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2349570244550705, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.924018859863281, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017191976308822632, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.294484257698059, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.46924257278442383, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.521272897720337, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.7060439586639404, "metrics/particle/momentum/class_4/event/energy_bias": -1.4954919815063477, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.300104856491089, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.20654296875, "metrics/particle/momentum/class_4/event/px_bias": 0.11564899981021881, "metrics/particle/momentum/class_4/event/py_bias": 0.02393948845565319, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0752968788146973, "metrics/particle/momentum/class_4/event/pz_bias": 0.6873370409011841, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.6002349853515625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01857774145901203, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17938758432865143, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17653778195381165, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03726194053888321, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21317127346992493, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.023622047156095505, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8291642665863037, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.20858655869960785, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.1761796474456787, "metrics/particle/momentum/class_5/accepted/samples": 127.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06297467648983002, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2795168459415436, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.28039854764938354, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05337085574865341, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1548387110233307, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4596797227859497, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06451612710952759, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.627225399017334, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.030003855004906654, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.1620659828186035, "metrics/particle/momentum/class_5/assigned/samples": 155.0, "metrics/particle/momentum/class_5/efficiency": 0.7987421154975891, "metrics/particle/momentum/class_5/event/energy_bias": -1.3820866346359253, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0109751224517822, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2170333862304688, "metrics/particle/momentum/class_5/event/px_bias": -0.2601548433303833, "metrics/particle/momentum/class_5/event/py_bias": 0.48767831921577454, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6543337106704712, "metrics/particle/momentum/class_5/event/pz_bias": 0.12245501577854156, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.080446243286133, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4192620515823364, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42136263847351074, "metrics/set_slot/angular_recall_all_dr0p10": 0.9433860778808594, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8017396926879883, "metrics/set_slot/soft_count_bias_mean": -0.5161401629447937}, "step_train_time": 5.210695266723633, "step_valid_time": 14648.093703746796, "step_total_time": 14653.304399251938, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0133715914261043, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17088871808816453, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2999, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2827, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2786, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339279814998348, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.942647549183061, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9382675074676402, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05735245081693898, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203832177073009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.928976325441814, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9246598075008298, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339279814998348, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000200058690941, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15755678514011007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2105, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9350132625994695, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9471562919838782, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9410456062291435, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05284370801612181, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9305923961096375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9426780116435289, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9365962180200222, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9350132625994695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052322939577207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16037928176587635, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1267, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1177, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1171, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9138198757763976, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9289660615627466, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9213307240704501, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07103393843725335, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9091614906832298, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9242304656669298, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9166340508806262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9138198757763976} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_42000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..933cb3037e66600033a101c918596d00ed25f142 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9897669130187607, "train": {"Classification_binary": 0.22916531562805176, "Cardinality": 0.1538309007883072, "Classification": 0.10027627646923065, "Regression_pt": 0.18975645303726196, "Regression_eta": 0.001765990862622857, "Regression_sin_phi": 0.0011088066967204213, "Regression_cos_phi": 0.0011319639161229134, "Regression_energy": 0.18753840029239655, "HitGrouping": 0.24566397070884705, "Auxiliary": 0.3116125166416168, "Total": 1.4171555042266846, "Total_unweighted": 0.7107432069024071}, "train_group": {}, "valid": {"Classification_binary": 0.19744515419006348, "Cardinality": 0.13982569177945456, "Classification": 0.09556442499160767, "Regression_pt": 0.19772350788116455, "Regression_eta": 0.0018708033797641594, "Regression_sin_phi": 0.001060621424888571, "Regression_cos_phi": 0.0010589223044613998, "Regression_energy": 0.1969215472539266, "HitGrouping": 0.24369625250498453, "Auxiliary": 0.29021795590718585, "Total": 1.3480981190999348, "Total_unweighted": 0.6916449814258764, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7139037847518921, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.32821547985076904, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.45244166254997253, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3174270987510681, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.32285076379776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6439973711967468, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6532572507858276, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2743735909461975, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4296848475933075, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2604207694530487, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.30673784017562866, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7160284519195557, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7354350090026855, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3295612633228302, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.425317645072937, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.32108592987060547, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.32011035084724426, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6070137619972229, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7211353778839111, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3241780698299408, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.43443769216537476, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.31446710228919983, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3197927176952362, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6286929845809937, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025597941130399704, "metrics/hit_grouping/calorimeter/background_recall": 0.6294038891792297, "metrics/hit_grouping/calorimeter/completeness": 0.3670814335346222, "metrics/hit_grouping/calorimeter/energy_completeness": 0.39116421341896057, "metrics/hit_grouping/calorimeter/energy_purity": 0.3866226077079773, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5146179795265198, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7411757111549377, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3405466377735138, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3698417544364929, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3331308960914612, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.344546377658844, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.584114670753479, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7144568562507629, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3378104269504547, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4774381220340729, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3252003490924835, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3282635509967804, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.64422607421875, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.685676097869873, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.27532508969306946, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5263556838035583, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2637769877910614, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2585698962211609, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7068638801574707, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7021490931510925, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3431299924850464, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5204609036445618, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3320782780647278, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3369610011577606, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.667517900466919, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7188591957092285, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4166136085987091, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6446886658668518, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.40624821186065674, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4045393168926239, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.633176326751709, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7418380975723267, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2919752597808838, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.31505289673805237, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.28378939628601074, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2904913127422333, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5863800644874573, "metrics/hit_grouping/calorimeter/purity": 0.36160388588905334, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8507081866264343, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06737634539604187, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16153846681118011, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06468804180622101, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.02715659886598587, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.06923077255487442, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7538938522338867, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1335907131433487, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12599772214889526, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13229846954345703, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1257038414478302, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4389965832233429, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7164893746376038, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.34061339497566223, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4579944908618927, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.330268532037735, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.33719319105148315, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6515280604362488, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7170446515083313, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4351464509963989, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.42110675573349, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4250084161758423, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6273161768913269, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7584171891212463, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30846887826919556, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4338594973087311, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4643012583255768, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5176668763160706, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6803808808326721, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43585097789764404, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38359278440475464, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3782183825969696, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6728361248970032, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6985413432121277, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4249007999897003, "metrics/hit_grouping/tracker/all/object_completeness": 0.39968329668045044, "metrics/hit_grouping/tracker/all/object_purity": 0.4002571403980255, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6453806757926941, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004630067385733128, "metrics/hit_grouping/tracker/background_recall": 0.02493632584810257, "metrics/hit_grouping/tracker/completeness": 0.41965243220329285, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5302656292915344, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7031087279319763, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4196811020374298, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45381343364715576, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4645582139492035, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6518450975418091, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.693308413028717, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43632209300994873, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3782126009464264, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3767000734806061, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6459166407585144, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6939623355865479, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.424352765083313, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3023344874382019, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.28254687786102295, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6304611563682556, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.722790002822876, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44518330693244934, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.41647759079933167, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.40771058201789856, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6071171760559082, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7651283740997314, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4513981342315674, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.44655880331993103, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4016023576259613, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.510652482509613, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6552432179450989, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39381876587867737, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3695622980594635, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.390310674905777, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7189182639122009, "metrics/hit_grouping/tracker/purity": 0.39989396929740906, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8867268562316895, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.010860121808946133, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.023482009768486023, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02659342624247074, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08079930394887924, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6715721487998962, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.17884914577007294, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1879536211490631, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20310063660144806, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5837221145629883, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6760409474372864, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5091789364814758, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47491154074668884, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4734812080860138, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7291187047958374, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.41554051637649536, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5466291308403015, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6021896600723267, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5825263857841492, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07018683850765228, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05172159895300865, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.20586585998535, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2836924195289612, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15325258672237396, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8143450617790222, "metrics/particle/class_1/pid_accuracy": 0.9564594030380249, "metrics/particle/class_2/efficiency": 0.38041284680366516, "metrics/particle/class_2/pid_accuracy": 0.5608965158462524, "metrics/particle/class_3/efficiency": 0.6759916543960571, "metrics/particle/class_3/pid_accuracy": 0.9555062055587769, "metrics/particle/class_4/efficiency": 0.6650625467300415, "metrics/particle/class_4/pid_accuracy": 0.33863964676856995, "metrics/particle/class_5/efficiency": 0.8399999737739563, "metrics/particle/class_5/pid_accuracy": 0.35476189851760864, "metrics/particle/count/bias_mean": -4.396484375, "metrics/particle/count/mae": 5.354817867279053, "metrics/particle/count/prediction_mean": 65.65950775146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08219151198863983, "metrics/particle/event/energy_response_mean": 0.9632644057273865, "metrics/particle/event/met_abs_error": 7.974786758422852, "metrics/particle/event/scalar_pt_relative_abs_error": 0.066278375685215, "metrics/particle/event/scalar_pt_response_mean": 0.9771086573600769, "metrics/particle/event/vector_pt_closure": 0.06583093851804733, "metrics/particle/matched/delta_eta_abs_mean": 0.01564769819378853, "metrics/particle/matched/delta_phi_abs_mean": 0.015695130452513695, "metrics/particle/matched/delta_r_mean": 0.024818696081638336, "metrics/particle/matched/energy_relative_abs_mean": 0.17310579121112823, "metrics/particle/matched/pid_accuracy": 0.9298811554908752, "metrics/particle/matched/pt_relative_abs_mean": 0.17018508911132812, "metrics/particle/matching/duplicate_fraction": 0.06481710821390152, "metrics/particle/matching/efficiency": 0.7171440124511719, "metrics/particle/matching/f1": 0.740375816822052, "metrics/particle/matching/purity": 0.7651631832122803, "metrics/particle/matching/target_coverage_dr0p05": 0.6763377785682678, "metrics/particle/matching/target_coverage_dr0p10": 0.7925301790237427, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4816367030143738, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7597624659538269, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02384849265217781, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17180366814136505, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16944196820259094, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010871730744838715, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21032312512397766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010912437923252583, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8930677771568298, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04340650141239166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2734432220458984, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7606.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.31226685643196106, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.6115975379943848, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.700877070426941, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5097227096557617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.18376241624355316, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 20.102252960205078, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02219744585454464, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.347893476486206, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.11334745585918427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6855039596557617, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9866.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6955010890960693, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.211484909057617, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.610054016113281, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.285024642944336, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.23463498055934906, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.37030088901519775, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.230939865112305, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.4358283281326294, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.039265632629395, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024885792285203934, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1717199683189392, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16926096379756927, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0011074573267251253, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21257486939430237, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03208868205547333, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6794308423995972, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.12084414064884186, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.130807399749756, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1714.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4702666103839874, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8633262515068054, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.8681998252868652, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.6753627061843872, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2077867090702057, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.34469223022461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06347862631082535, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.773648977279663, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0731174573302269, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.495377063751221, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2363.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5673617720603943, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.57731294631958, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.297405242919922, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5682497024536133, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14201557636260986, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.042288973927497864, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.1133394241333, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1086292266845703, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.277944564819336, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024962162598967552, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17230817675590515, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16904425621032715, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01332850195467472, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21004579961299896, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009366804151795805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2989320456981659, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.001419651205651462, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8786899447441101, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16014.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2494024783372879, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4857662916183472, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1854465007781982, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9941859841346741, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.179958313703537, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.164775848388672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005332816392183304, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5376163125038147, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.001955341314896941, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.965858817100525, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20627.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7470261454582214, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0199230909347534, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.47451400756836, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.782135009765625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3078892230987549, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.8327375650405884, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.767827987670898, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1080547571182251, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.81501579284668, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024622637778520584, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17211690545082092, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1691783368587494, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011764079332351685, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21030110120773315, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006039314903318882, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5707079768180847, "metrics/particle/momentum/all/accepted/pz_bias": 0.02031034789979458, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.787247657775879, "metrics/particle/momentum/all/accepted/samples": 25334.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2841639220714569, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4787850379943848, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3893234729766846, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1979814767837524, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18310202658176422, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.041696548461914, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01457876805216074, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9417415857315063, "metrics/particle/momentum/all/assigned/pz_bias": 0.038067009299993515, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2473669052124023, "metrics/particle/momentum/all/assigned/samples": 32856.0, "metrics/particle/momentum/all/efficiency": 0.7157710194587708, "metrics/particle/momentum/all/event/energy_bias": -9.768875122070312, "metrics/particle/momentum/all/event/p3_error_mean": 22.30221939086914, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.981388092041016, "metrics/particle/momentum/all/event/px_bias": -0.684539794921875, "metrics/particle/momentum/all/event/py_bias": 1.1607495546340942, "metrics/particle/momentum/all/event/pz_abs_mean": 16.671077728271484, "metrics/particle/momentum/all/event/pz_bias": 0.21914434432983398, "metrics/particle/momentum/all/event/pz_residual_rms": 26.60569953918457, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026625636965036392, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17124466598033905, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16792963445186615, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01256132498383522, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20897240936756134, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007065057288855314, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6293171048164368, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01792697235941887, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7928951978683472, "metrics/particle/momentum/class_1/accepted/samples": 13588.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14561207592487335, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.43272432684898376, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4981604516506195, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3222239017486572, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1276041716337204, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.054943561553955, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010974702425301075, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8265277743339539, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0342637337744236, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.803004503250122, "metrics/particle/momentum/class_1/assigned/samples": 16128.0, "metrics/particle/momentum/class_1/efficiency": 0.8142865896224976, "metrics/particle/momentum/class_1/event/energy_bias": 7.376572132110596, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.009321212768555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.790891647338867, "metrics/particle/momentum/class_1/event/px_bias": -1.705244779586792, "metrics/particle/momentum/class_1/event/py_bias": 1.2553346157073975, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.410292625427246, "metrics/particle/momentum/class_1/event/pz_bias": 0.04605978727340698, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.571197509765625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03418929502367973, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2053588181734085, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20594675838947296, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04215513914823532, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24400386214256287, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03035714291036129, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8971457481384277, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.29985180497169495, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.5212178230285645, "metrics/particle/momentum/class_2/accepted/samples": 1120.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7224997282028198, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8002410531044006, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9410079717636108, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.522348940372467, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3427477180957794, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.701643466949463, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05265677347779274, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7197840213775635, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.17665617167949677, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.076809883117676, "metrics/particle/momentum/class_2/assigned/samples": 2089.0, "metrics/particle/momentum/class_2/efficiency": 0.37621766328811646, "metrics/particle/momentum/class_2/event/energy_bias": -17.707088470458984, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.089975357055664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.767791748046875, "metrics/particle/momentum/class_2/event/px_bias": 1.490753173828125, "metrics/particle/momentum/class_2/event/py_bias": -1.7375998497009277, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.495426177978516, "metrics/particle/momentum/class_2/event/pz_bias": 0.3928408920764923, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.164430618286133, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0209057554602623, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1689823567867279, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16637612879276276, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016276739537715912, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20744676887989044, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017578124534338713, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.32074612379074097, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0055751861073076725, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0719245672225952, "metrics/particle/momentum/class_3/accepted/samples": 10240.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.38031989336013794, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.7733514308929443, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.477811574935913, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.301711082458496, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2226070910692215, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.872196197509766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012724445201456547, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7884012460708618, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018480723723769188, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0906312465667725, "metrics/particle/momentum/class_3/assigned/samples": 14146.0, "metrics/particle/momentum/class_3/efficiency": 0.6733741164207458, "metrics/particle/momentum/class_3/event/energy_bias": 3.820237874984741, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.132209777832031, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.627407073974609, "metrics/particle/momentum/class_3/event/px_bias": -0.11258317530155182, "metrics/particle/momentum/class_3/event/py_bias": 0.9387624263763428, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.203804969787598, "metrics/particle/momentum/class_3/event/pz_bias": -0.24546533823013306, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.89494514465332, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026696456596255302, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18648315966129303, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1756903976202011, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.049985095858573914, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22079256176948547, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00401606410741806, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9887450337409973, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.14182230830192566, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9826127290725708, "metrics/particle/momentum/class_4/accepted/samples": 249.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26746582984924316, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8873436450958252, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5711421966552734, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4347715377807617, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23235294222831726, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.857605457305908, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0117647061124444, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1701056957244873, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2758878767490387, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2755844593048096, "metrics/particle/momentum/class_4/assigned/samples": 340.0, "metrics/particle/momentum/class_4/efficiency": 0.6840659379959106, "metrics/particle/momentum/class_4/event/energy_bias": -1.6546939611434937, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6804614067077637, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9586284160614014, "metrics/particle/momentum/class_4/event/px_bias": -0.14058837294578552, "metrics/particle/momentum/class_4/event/py_bias": 0.1972254514694214, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5081788301467896, "metrics/particle/momentum/class_4/event/pz_bias": 0.08110418915748596, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.625555038452148, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021798687055706978, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19504769146442413, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19004975259304047, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03327331691980362, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23496884107589722, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029197080060839653, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8373205661773682, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3146507740020752, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.423692464828491, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05105973407626152, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.41015559434890747, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.40658196806907654, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.16366009414196014, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09150326997041702, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.148716926574707, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05228758230805397, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.479954957962036, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.17085346579551697, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.017543792724609, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.8616352081298828, "metrics/particle/momentum/class_5/event/energy_bias": -1.6039021015167236, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.9984068870544434, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.167802333831787, "metrics/particle/momentum/class_5/event/px_bias": -0.2168765515089035, "metrics/particle/momentum/class_5/event/py_bias": 0.5070268511772156, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6762595176696777, "metrics/particle/momentum/class_5/event/pz_bias": -0.055395208299160004, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.115385055541992, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4447510838508606, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.44538938999176025, "metrics/set_slot/angular_recall_all_dr0p10": 0.9380424618721008, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7867497801780701, "metrics/set_slot/soft_count_bias_mean": -0.7992399334907532}, "step_train_time": 4.695621490478516, "step_valid_time": 17158.013430595398, "step_total_time": 17162.709052324295, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9935974335495156, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17623588091003883, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2809, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2768, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9279814998348199, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9335327351279494, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9307488402915838, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06646726487205051, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9144367360422861, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9199069458291791, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9171636845593109, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9279814998348199, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9812154768330505, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16553316374292293, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2229, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2088, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2082, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9230769230769231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9367429340511441, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9298597194388778, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.063257065948856, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9204244031830239, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9340511440107672, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.927187708750835, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9230769230769231, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9809125072897276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16516379509742352, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1270, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1171, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9091614906832298, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9220472440944882, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9155590304925724, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07795275590551182, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9021739130434783, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9149606299212598, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.90852228303362, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9091614906832298} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_43000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9a55981ee717aa08ad8470f8a815d9cae8aab1 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0371423157096835, "train": {"Classification_binary": 0.1851489543914795, "Cardinality": 0.13167870044708252, "Classification": 0.08739826083183289, "Regression_pt": 0.1704798936843872, "Regression_eta": 0.0015009068883955479, "Regression_sin_phi": 0.0008885852294042706, "Regression_cos_phi": 0.0008677783189341426, "Regression_energy": 0.1695600152015686, "HitGrouping": 0.2276415228843689, "Auxiliary": 0.2613914906978607, "Total": 1.2106198072433472, "Total_unweighted": 0.6158443945460021}, "train_group": {}, "valid": {"Classification_binary": 0.19038321574529013, "Cardinality": 0.1324609915415446, "Classification": 0.09181423981984456, "Regression_pt": 0.1781687537829081, "Regression_eta": 0.0016570974451800187, "Regression_sin_phi": 0.0009694580609599749, "Regression_cos_phi": 0.0009841867722570896, "Regression_energy": 0.1774027943611145, "HitGrouping": 0.23241070906321207, "Auxiliary": 0.2745660940806071, "Total": 1.2652877966562908, "Total_unweighted": 0.6413797459875544, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7437546253204346, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.35131463408470154, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42948898673057556, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3431002199649811, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3458573520183563, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5856238603591919, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6829317212104797, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2964279353618622, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4144357740879059, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.28481414914131165, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.31827229261398315, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6706201434135437, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7589057683944702, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3552224636077881, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40636298060417175, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.34766775369644165, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34244316816329956, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5531453490257263, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7472156882286072, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.34869134426116943, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41452887654304504, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3405602276325226, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3413738012313843, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5741337537765503, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026103561744093895, "metrics/hit_grouping/calorimeter/background_recall": 0.634949803352356, "metrics/hit_grouping/calorimeter/completeness": 0.4014972150325775, "metrics/hit_grouping/calorimeter/energy_completeness": 0.42400065064430237, "metrics/hit_grouping/calorimeter/energy_purity": 0.41907674074172974, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4957929849624634, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.765975832939148, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3637758195400238, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3427889049053192, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.35782405734062195, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3656611740589142, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5254105925559998, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7422088384628296, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.361514687538147, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4592430889606476, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3514530658721924, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.35137322545051575, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5970402956008911, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7127240896224976, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3033991754055023, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5197826027870178, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2926670014858246, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2795450985431671, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6533939838409424, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7342373728752136, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3756416440010071, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5061351656913757, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3661465644836426, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.36215338110923767, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6079726219177246, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7400816082954407, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.452987402677536, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6195709109306335, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.44541579484939575, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.42743077874183655, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5860806107521057, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7620809674263, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3063719570636749, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2898084819316864, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2996356189250946, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.30802232027053833, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5362260341644287, "metrics/hit_grouping/calorimeter/purity": 0.3957114517688751, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8403039574623108, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07292051613330841, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16769230365753174, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07472161203622818, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03197953477501869, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0615384615957737, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7463605999946594, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1392073631286621, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11060433089733124, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.13695842027664185, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14001239836215973, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4047890603542328, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7452807426452637, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3666178286075592, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.43727046251296997, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.35793671011924744, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.35954979062080383, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5947993397712708, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7176641225814819, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4348475933074951, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.42878925800323486, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.43586739897727966, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6240286827087402, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7695500254631042, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3381940424442291, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.43809080123901367, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4411516487598419, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.49018508195877075, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6734146475791931, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44254547357559204, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3904882073402405, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3957037031650543, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6789005398750305, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6954969763755798, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43132439255714417, "metrics/hit_grouping/tracker/all/object_completeness": 0.4066031873226166, "metrics/hit_grouping/tracker/all/object_purity": 0.4122259318828583, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6456640958786011, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005795209668576717, "metrics/hit_grouping/tracker/background_recall": 0.03465024009346962, "metrics/hit_grouping/tracker/completeness": 0.42603883147239685, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5250058770179749, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7016155123710632, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4206833839416504, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45703160762786865, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4742701053619385, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6461958885192871, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6898478865623474, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4547877311706543, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3881969451904297, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3878946602344513, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6550542712211609, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6879169940948486, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4300161898136139, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31421545147895813, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3003484308719635, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6345064640045166, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7207680940628052, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4570452868938446, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42844972014427185, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4224482774734497, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6070272922515869, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7545145153999329, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46804261207580566, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45717206597328186, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4190661907196045, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5392809510231018, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6521979570388794, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.391360342502594, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.36925092339515686, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39770612120628357, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7147037982940674, "metrics/hit_grouping/tracker/purity": 0.40645545721054077, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8846642374992371, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.016507385298609734, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.028171883895993233, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03025609441101551, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08644656836986542, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6758309602737427, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20839813351631165, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2054133415222168, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20831221342086792, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5795748829841614, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6720317006111145, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5130748152732849, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4809343218803406, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4873480498790741, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.729177713394165, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4183533787727356, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5424293875694275, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5984302759170532, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5908891558647156, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06737397611141205, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06008436530828476, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.210073471069336, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29807576537132263, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1676359623670578, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8394736647605896, "metrics/particle/class_1/pid_accuracy": 0.9617696404457092, "metrics/particle/class_2/efficiency": 0.400942325592041, "metrics/particle/class_2/pid_accuracy": 0.592893123626709, "metrics/particle/class_3/efficiency": 0.7050179243087769, "metrics/particle/class_3/pid_accuracy": 0.9582509398460388, "metrics/particle/class_4/efficiency": 0.6544754505157471, "metrics/particle/class_4/pid_accuracy": 0.33529412746429443, "metrics/particle/class_5/efficiency": 0.8500000238418579, "metrics/particle/class_5/pid_accuracy": 0.4000000059604645, "metrics/particle/count/bias_mean": -3.1100261211395264, "metrics/particle/count/mae": 4.597005367279053, "metrics/particle/count/prediction_mean": 66.94596099853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.0789194405078888, "metrics/particle/event/energy_response_mean": 0.9716623425483704, "metrics/particle/event/met_abs_error": 7.873069763183594, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0640212669968605, "metrics/particle/event/scalar_pt_response_mean": 0.9879024028778076, "metrics/particle/event/vector_pt_closure": 0.06465333700180054, "metrics/particle/matched/delta_eta_abs_mean": 0.014456935226917267, "metrics/particle/matched/delta_phi_abs_mean": 0.015106349252164364, "metrics/particle/matched/delta_r_mean": 0.02342899516224861, "metrics/particle/matched/energy_relative_abs_mean": 0.16666504740715027, "metrics/particle/matched/pid_accuracy": 0.9355250000953674, "metrics/particle/matched/pt_relative_abs_mean": 0.16600018739700317, "metrics/particle/matching/duplicate_fraction": 0.06784078478813171, "metrics/particle/matching/efficiency": 0.7431648969650269, "metrics/particle/matching/f1": 0.7600351572036743, "metrics/particle/matching/purity": 0.7776891589164734, "metrics/particle/matching/target_coverage_dr0p05": 0.7050629258155823, "metrics/particle/matching/target_coverage_dr0p10": 0.8129844069480896, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.507146418094635, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7826329469680786, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022688060998916626, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1653522253036499, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16405370831489563, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00890658050775528, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2050936222076416, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010722491890192032, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8702309131622314, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.020860308781266212, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.306326389312744, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7834.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.30472180247306824, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2377632856369019, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4622244834899902, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2788679599761963, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1774241328239441, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.657674789428711, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022931698709726334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3354228734970093, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05908510088920593, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8091437816619873, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10117.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7163496613502502, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.1275858879089355, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.486627578735352, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.217087268829346, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.059252381324768066, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0950177013874054, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.105411529541016, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1401406526565552, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.46631908416748, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023862898349761963, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16962043941020966, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1702873259782791, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018229864537715912, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21172870695590973, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.035026270896196365, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7721831798553467, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06108366325497627, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.275115966796875, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1713.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5177261829376221, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1521234512329102, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.5691559314727783, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.3843555450439453, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23121149837970734, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.759737014770508, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07351129502058029, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9105398654937744, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1323472112417221, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.9818115234375, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2435.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5670307874679565, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.207156181335449, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.177051544189453, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.389441967010498, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14417248964309692, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15993930399417877, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.010741233825684, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7656804323196411, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.920494079589844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023375442251563072, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16446827352046967, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1642441302537918, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01251054834574461, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20551681518554688, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000838473963085562, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2803947329521179, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 2.184752156608738e-05, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8226664066314697, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16697.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22961486876010895, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.365139365196228, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0805696249008179, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8978274464607239, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1598595231771469, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.190465927124023, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00543608982115984, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5144588351249695, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010452843271195889, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0268499851226807, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20787.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7788869738578796, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.1475449651479721, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.3876953125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.763945579528809, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.23104102909564972, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6675975322723389, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.722150802612305, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.18611906468868256, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.342799186706543, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023202070966362953, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16506841778755188, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16458174586296082, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011808051727712154, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20580200850963593, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006020423490554094, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5538366436958313, "metrics/particle/momentum/all/accepted/pz_bias": 0.002253778977319598, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.792019009590149, "metrics/particle/momentum/all/accepted/samples": 26244.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2734496593475342, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3109278678894043, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3051085472106934, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1220297813415527, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17040103673934937, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.467673301696777, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01571732759475708, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9385913014411926, "metrics/particle/momentum/all/assigned/pz_bias": 0.014780943281948566, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.244187116622925, "metrics/particle/momentum/all/assigned/samples": 33339.0, "metrics/particle/momentum/all/efficiency": 0.741481602191925, "metrics/particle/momentum/all/event/energy_bias": -8.187196731567383, "metrics/particle/momentum/all/event/p3_error_mean": 20.95366859436035, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.603729248046875, "metrics/particle/momentum/all/event/px_bias": -0.4344658851623535, "metrics/particle/momentum/all/event/py_bias": 0.4126405119895935, "metrics/particle/momentum/all/event/pz_abs_mean": 15.505583763122559, "metrics/particle/momentum/all/event/pz_bias": -0.8116587400436401, "metrics/particle/momentum/all/event/pz_residual_rms": 24.397029876708984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024848464876413345, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16067259013652802, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1603820025920868, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009419267997145653, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20077237486839294, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006274509709328413, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6001121997833252, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.002286455826833844, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8038078546524048, "metrics/particle/momentum/class_1/accepted/samples": 14025.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12641824781894684, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.41987547278404236, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.491414338350296, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.32529667019844055, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10857529193162918, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.885014533996582, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011072834953665733, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8039616942405701, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.000783085823059082, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.753523349761963, "metrics/particle/momentum/class_1/assigned/samples": 16256.0, "metrics/particle/momentum/class_1/efficiency": 0.8404746055603027, "metrics/particle/momentum/class_1/event/energy_bias": 5.361435413360596, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.301403045654297, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.23801326751709, "metrics/particle/momentum/class_1/event/px_bias": -0.9138957262039185, "metrics/particle/momentum/class_1/event/py_bias": 1.1365550756454468, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.025035858154297, "metrics/particle/momentum/class_1/event/pz_bias": -0.9533454179763794, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.420337677001953, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03384445980191231, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20840150117874146, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2085057944059372, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.027699224650859833, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2492620199918747, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.034246575087308884, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9083224534988403, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03978569060564041, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.434564590454102, "metrics/particle/momentum/class_2/accepted/samples": 1168.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7724446654319763, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9670313596725464, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2336366176605225, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8111400008201599, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3524552583694458, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.4242634773254395, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06516750901937485, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.928682327270508, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10985992103815079, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.793998718261719, "metrics/particle/momentum/class_2/assigned/samples": 2179.0, "metrics/particle/momentum/class_2/efficiency": 0.392341285943985, "metrics/particle/momentum/class_2/event/energy_bias": -14.780613899230957, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.305131912231445, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.412238121032715, "metrics/particle/momentum/class_2/event/px_bias": 1.07357656955719, "metrics/particle/momentum/class_2/event/py_bias": -1.6535087823867798, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.99416732788086, "metrics/particle/momentum/class_2/event/pz_bias": 0.8916085958480835, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.295169830322266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0199061781167984, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16587170958518982, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16523559391498566, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019218791276216507, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2069844752550125, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002154566813260317, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3160935938358307, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.014012769795954227, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.088388442993164, "metrics/particle/momentum/class_3/accepted/samples": 10675.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36336997151374817, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.354672431945801, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2285239696502686, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0619986057281494, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21108178794384003, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.42535400390625, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012567698955535889, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7580893039703369, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.006319910287857056, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.84918212890625, "metrics/particle/momentum/class_3/assigned/samples": 14402.0, "metrics/particle/momentum/class_3/efficiency": 0.7019793391227722, "metrics/particle/momentum/class_3/event/energy_bias": 4.141794204711914, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.37319564819336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.729677677154541, "metrics/particle/momentum/class_3/event/px_bias": -0.13456593453884125, "metrics/particle/momentum/class_3/event/py_bias": 0.5201728940010071, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.94411849975586, "metrics/particle/momentum/class_3/event/pz_bias": -1.1267677545547485, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.44045639038086, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024626348167657852, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1675107479095459, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16328254342079163, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.027925770729780197, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20427586138248444, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016393441706895828, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2069398164749146, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4281310439109802, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6847176551818848, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.37429341673851013, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.313628911972046, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9953436851501465, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8562761545181274, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24347825348377228, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.773367404937744, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02028985507786274, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3740665912628174, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.34317052364349365, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.746213436126709, "metrics/particle/momentum/class_4/assigned/samples": 345.0, "metrics/particle/momentum/class_4/efficiency": 0.6703296899795532, "metrics/particle/momentum/class_4/event/energy_bias": -1.5251176357269287, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0589163303375244, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.124098777770996, "metrics/particle/momentum/class_4/event/px_bias": -0.1784762442111969, "metrics/particle/momentum/class_4/event/py_bias": 0.04189646616578102, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8505483865737915, "metrics/particle/momentum/class_4/event/pz_bias": 0.3603724241256714, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.445760726928711, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018013836815953255, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17921730875968933, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17166553437709808, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.013912996277213097, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21520496904850006, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.022727273404598236, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.671243667602539, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.19494903087615967, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.1035382747650146, "metrics/particle/momentum/class_5/accepted/samples": 132.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10152841359376907, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.39600077271461487, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.32418808341026306, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.09257791936397552, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15286624431610107, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7236435413360596, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08917197585105896, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.8588509559631348, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.19907039403915405, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.809807777404785, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.8301886916160583, "metrics/particle/momentum/class_5/event/energy_bias": -1.3846956491470337, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.923632860183716, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.177436590194702, "metrics/particle/momentum/class_5/event/px_bias": -0.28110453486442566, "metrics/particle/momentum/class_5/event/py_bias": 0.3675248920917511, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.548517107963562, "metrics/particle/momentum/class_5/event/pz_bias": 0.016473233699798584, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.937318801879883, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4163719713687897, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.41758814454078674, "metrics/set_slot/angular_recall_all_dr0p10": 0.944529116153717, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8052060008049011, "metrics/set_slot/soft_count_bias_mean": 0.5298304557800293}, "step_train_time": 5.728140592575073, "step_valid_time": 19753.061133623123, "step_total_time": 19758.789274692535, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994843806340069, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17763554156787076, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2995, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2815, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2784, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299636603898249, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9398998330550918, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9349053470607771, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06010016694490818, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9197224975222993, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9295492487479132, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9246097641979408, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299636603898249, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.994939506930211, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16053119068926358, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2100, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2095, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9283819628647215, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9333333333333333, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9308510638297872, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06666666666666667, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9261715296198055, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9311111111111111, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9286347517730497, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9283819628647215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9848806003571985, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15713552423287058, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1178, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1170, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9145962732919255, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9341792228390167, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9242840329540997, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06582077716098335, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9083850931677019, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9278350515463918, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9180070615927814, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9145962732919255} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_44000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..ab623702094c885cde5198f895f1dac1c8825c54 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0845177184006065, "train": {"Classification_binary": 0.19513654708862305, "Cardinality": 0.13974134624004364, "Classification": 0.07944411784410477, "Regression_pt": 0.17373378574848175, "Regression_eta": 0.001662290538661182, "Regression_sin_phi": 0.0010042578214779496, "Regression_cos_phi": 0.0009528179070912302, "Regression_energy": 0.1743963211774826, "HitGrouping": 0.23239518702030182, "Auxiliary": 0.2695709466934204, "Total": 1.261752724647522, "Total_unweighted": 0.6263301381259225}, "train_group": {}, "valid": {"Classification_binary": 0.187412699063619, "Cardinality": 0.13402249415715536, "Classification": 0.09068117539087932, "Regression_pt": 0.173629363377889, "Regression_eta": 0.0016484674997627735, "Regression_sin_phi": 0.0009199039389689764, "Regression_cos_phi": 0.0009291196862856547, "Regression_energy": 0.17291128635406494, "HitGrouping": 0.23066484928131104, "Auxiliary": 0.2693977157274882, "Total": 1.243541161219279, "Total_unweighted": 0.6281320153114697, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7407605051994324, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3637921214103699, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4327949285507202, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3548608422279358, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3595871031284332, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5969585180282593, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6862854957580566, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.32097089290618896, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41782447695732117, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.31226038932800293, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3396191895008087, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.664181649684906, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7587770819664001, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36230185627937317, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4109596014022827, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35490232706069946, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.35353705286979675, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5549013614654541, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7464772462844849, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3590741455554962, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4186018109321594, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35106417536735535, "metrics/hit_grouping/calorimeter/all/object_purity": 0.35423555970191956, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5782370567321777, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0257368516176939, "metrics/hit_grouping/calorimeter/background_recall": 0.6316802501678467, "metrics/hit_grouping/calorimeter/completeness": 0.40601494908332825, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4296131134033203, "metrics/hit_grouping/calorimeter/energy_purity": 0.42456135153770447, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.48426353931427, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7669628262519836, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.37708818912506104, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.350373238325119, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3715609908103943, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37819257378578186, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.526604950428009, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.743690013885498, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.37343424558639526, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4580300748348236, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36276909708976746, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.36837631464004517, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5953420400619507, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.706020712852478, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.30597984790802, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5219314694404602, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.29548147320747375, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.28879040479660034, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6697004437446594, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7321327328681946, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3833114802837372, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5031454563140869, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3750040531158447, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.377658873796463, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.613017737865448, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.725324273109436, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4441835582256317, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.622710645198822, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.43160533905029297, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.44238588213920593, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6169544458389282, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7646054625511169, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3221266567707062, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.30159369111061096, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.31502094864845276, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3177740275859833, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5358912348747253, "metrics/hit_grouping/calorimeter/purity": 0.4000134766101837, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8471404314041138, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.079115130007267, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1599999964237213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07967948168516159, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.036232247948646545, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05384615436196327, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7639123797416687, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14506113529205322, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11117445677518845, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14388880133628845, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1441592276096344, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38939565420150757, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7433289289474487, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.377350389957428, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4417956471443176, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.36876440048217773, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3730962574481964, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.600275456905365, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7137259244918823, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4320083558559418, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4378196895122528, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45411020517349243, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6198446154594421, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7759802341461182, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3381940424442291, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4682594835758209, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46999576687812805, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4800897240638733, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6723209619522095, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43624478578567505, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3948713541030884, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40723931789398193, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6800031661987305, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6941378116607666, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42664840817451477, "metrics/hit_grouping/tracker/all/object_completeness": 0.41462674736976624, "metrics/hit_grouping/tracker/all/object_purity": 0.4273391664028168, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6441526412963867, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00551459938287735, "metrics/hit_grouping/tracker/background_recall": 0.026890955865383148, "metrics/hit_grouping/tracker/completeness": 0.43385300040245056, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5088313817977905, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7054272890090942, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41949886083602905, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47130706906318665, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.48740580677986145, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6395444273948669, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6858187913894653, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4422234892845154, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.393207311630249, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4094882309436798, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6573386788368225, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6779158115386963, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4259708821773529, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31156250834465027, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31296631693840027, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6403721570968628, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7149043083190918, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4473400413990021, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43370065093040466, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4427078664302826, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6152947545051575, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7319950461387634, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4434087872505188, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.443210244178772, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4395865797996521, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5645805597305298, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6604277491569519, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.396745502948761, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38475242257118225, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40516433119773865, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.695738673210144, "metrics/hit_grouping/tracker/purity": 0.4137936234474182, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.877447783946991, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.017376193776726723, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.027250168845057487, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03171194717288017, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.09079061448574066, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6683024764060974, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20373250544071198, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20846575498580933, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22039592266082764, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.586832582950592, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6721707582473755, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5076441764831543, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49073946475982666, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5046620965003967, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7258721590042114, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4094579517841339, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5398077368736267, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5945907235145569, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5987764596939087, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07626941800117493, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06797166168689728, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.906707763671875, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2967844009399414, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16634458303451538, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8504946827888489, "metrics/particle/class_1/pid_accuracy": 0.9609622359275818, "metrics/particle/class_2/efficiency": 0.40453219413757324, "metrics/particle/class_2/pid_accuracy": 0.6128674149513245, "metrics/particle/class_3/efficiency": 0.7122476696968079, "metrics/particle/class_3/pid_accuracy": 0.9620180726051331, "metrics/particle/class_4/efficiency": 0.676612138748169, "metrics/particle/class_4/pid_accuracy": 0.35561877489089966, "metrics/particle/class_5/efficiency": 0.8600000143051147, "metrics/particle/class_5/pid_accuracy": 0.42790699005126953, "metrics/particle/count/bias_mean": -2.7669270038604736, "metrics/particle/count/mae": 4.421875, "metrics/particle/count/prediction_mean": 67.2890625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07596208900213242, "metrics/particle/event/energy_response_mean": 0.9865373969078064, "metrics/particle/event/met_abs_error": 8.232455253601074, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06363574415445328, "metrics/particle/event/scalar_pt_response_mean": 0.9993347525596619, "metrics/particle/event/vector_pt_closure": 0.06603113561868668, "metrics/particle/matched/delta_eta_abs_mean": 0.014308501034975052, "metrics/particle/matched/delta_phi_abs_mean": 0.014605189673602581, "metrics/particle/matched/delta_r_mean": 0.02289687842130661, "metrics/particle/matched/energy_relative_abs_mean": 0.16372810304164886, "metrics/particle/matched/pid_accuracy": 0.9377927184104919, "metrics/particle/matched/pt_relative_abs_mean": 0.1625591367483139, "metrics/particle/matching/duplicate_fraction": 0.06900421530008316, "metrics/particle/matching/efficiency": 0.7520305514335632, "metrics/particle/matching/f1": 0.7671808004379272, "metrics/particle/matching/purity": 0.7829540371894836, "metrics/particle/matching/target_coverage_dr0p05": 0.718482255935669, "metrics/particle/matching/target_coverage_dr0p10": 0.8194617629051208, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5214858055114746, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.790931761264801, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02230008691549301, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16255877912044525, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16200141608715057, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011683856137096882, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20282655954360962, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008754624053835869, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8341753482818604, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.014621522277593613, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1982789039611816, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8110.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28418993949890137, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4267727136611938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4994980096817017, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3212456703186035, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1641981303691864, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.285968780517578, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019911721348762512, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.311255693435669, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0034426867496222258, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7464816570281982, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10195.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7415874004364014, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3228861391544342, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.674674987792969, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.404350280761719, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.054998595267534256, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5505021810531616, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.25777816772461, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7704508304595947, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.338664054870605, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02334875427186489, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1679711788892746, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1679922640323639, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.008776047267019749, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21028625965118408, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.032082654535770416, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7112071514129639, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.009055317379534245, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.275201797485352, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1839.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5147788524627686, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2787622213363647, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.868598222732544, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6814963817596436, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21374651789665222, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.238014221191406, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06912991404533386, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.956437110900879, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1477821320295334, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.048416137695312, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2517.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6087388396263123, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.377915859222412, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.264518737792969, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4066038131713867, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.02447422221302986, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05391275882720947, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.099336624145508, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.9697396755218506, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.18836212158203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023109426721930504, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16438958048820496, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16283440589904785, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016458118334412575, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2029666006565094, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008955223602242768, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27818354964256287, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0021202056668698788, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.821855366230011, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16750.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22854465246200562, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4392844438552856, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1016278266906738, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9225432872772217, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.156524658203125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.474350929260254, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006099611055105925, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5231958627700806, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01788792945444584, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0195794105529785, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20821.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7813593149185181, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7435328364372253, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.752057075500488, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.003473281860352, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4024617075920105, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.9151103496551514, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.866178035736084, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4204711318016052, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.955456733703613, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02288006991147995, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16408014297485352, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16293667256832123, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014478771947324276, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20343679189682007, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005430914927273989, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5457749366760254, "metrics/particle/momentum/all/accepted/pz_bias": 0.006395247299224138, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7749814987182617, "metrics/particle/momentum/all/accepted/samples": 26699.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26694726943969727, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.423431634902954, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3552212715148926, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1757879257202148, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16315270960330963, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.558416366577148, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015029970556497574, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9454288482666016, "metrics/particle/momentum/all/assigned/pz_bias": -0.021152717992663383, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4149792194366455, "metrics/particle/momentum/all/assigned/samples": 33533.0, "metrics/particle/momentum/all/efficiency": 0.7543368935585022, "metrics/particle/momentum/all/event/energy_bias": -3.9572689533233643, "metrics/particle/momentum/all/event/p3_error_mean": 22.170696258544922, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.468765258789062, "metrics/particle/momentum/all/event/px_bias": -0.43298614025115967, "metrics/particle/momentum/all/event/py_bias": 2.5195252895355225, "metrics/particle/momentum/all/event/pz_abs_mean": 16.12201499938965, "metrics/particle/momentum/all/event/pz_bias": -3.1606619358062744, "metrics/particle/momentum/all/event/pz_residual_rms": 25.229055404663086, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024412691593170166, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15978029370307922, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15857234597206116, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.017575614154338837, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1981247067451477, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005896805785596371, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5943141579627991, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.004456601571291685, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8003087043762207, "metrics/particle/momentum/class_1/accepted/samples": 14245.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12228837609291077, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3822181820869446, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.44520214200019836, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2870449423789978, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0976744219660759, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.7166848182678223, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010220318101346493, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7704868316650391, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01647365279495716, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4465835094451904, "metrics/particle/momentum/class_1/assigned/samples": 16340.0, "metrics/particle/momentum/class_1/efficiency": 0.8536585569381714, "metrics/particle/momentum/class_1/event/energy_bias": 5.997530937194824, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.788928985595703, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.915275573730469, "metrics/particle/momentum/class_1/event/px_bias": -0.31593620777130127, "metrics/particle/momentum/class_1/event/py_bias": 1.3553013801574707, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.633719444274902, "metrics/particle/momentum/class_1/event/pz_bias": 0.3293614089488983, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.18144989013672, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03300125524401665, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20063713192939758, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2006264626979828, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.021732967346906662, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2412085086107254, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029190992936491966, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8369698524475098, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09769868850708008, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.268406867980957, "metrics/particle/momentum/class_2/accepted/samples": 1199.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7887440919876099, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9717865586280823, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2517585754394531, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8495762944221497, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3459990918636322, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.884912014007568, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06079570949077606, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9584732055664062, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09861074388027191, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.981922626495361, "metrics/particle/momentum/class_2/assigned/samples": 2237.0, "metrics/particle/momentum/class_2/efficiency": 0.40275445580482483, "metrics/particle/momentum/class_2/event/energy_bias": -10.762495040893555, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.05599594116211, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.371319770812988, "metrics/particle/momentum/class_2/event/px_bias": 0.48030778765678406, "metrics/particle/momentum/class_2/event/py_bias": -0.2042112499475479, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.656759262084961, "metrics/particle/momentum/class_2/event/pz_bias": -1.670969843864441, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.307111740112305, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019745949655771255, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16528825461864471, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16420811414718628, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01425584964454174, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20551697909832, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021184489596635103, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.312467485666275, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008184514939785004, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1098273992538452, "metrics/particle/momentum/class_3/accepted/samples": 10857.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35186412930488586, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.6247856616973877, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.39082670211792, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.220928430557251, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20826126635074615, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.606943130493164, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012869300320744514, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8065357208251953, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.052235882729291916, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.480708122253418, "metrics/particle/momentum/class_3/assigned/samples": 14453.0, "metrics/particle/momentum/class_3/efficiency": 0.7139475345611572, "metrics/particle/momentum/class_3/event/energy_bias": 4.1272101402282715, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.782255172729492, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.042633056640625, "metrics/particle/momentum/class_3/event/px_bias": -0.7032324075698853, "metrics/particle/momentum/class_3/event/py_bias": 0.9076908826828003, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.07492446899414, "metrics/particle/momentum/class_3/event/pz_bias": -2.2179508209228516, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.014644622802734, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026200711727142334, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17257559299468994, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16860736906528473, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03587631508708, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2072714865207672, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01149425283074379, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.051527500152588, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2678087055683136, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.484705686569214, "metrics/particle/momentum/class_4/accepted/samples": 261.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2613289952278137, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.817950963973999, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2120652198791504, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0847272872924805, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.20749279856681824, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.753993034362793, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020172910764813423, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2305883169174194, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.015123625285923481, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.596602439880371, "metrics/particle/momentum/class_4/assigned/samples": 347.0, "metrics/particle/momentum/class_4/efficiency": 0.7170329689979553, "metrics/particle/momentum/class_4/event/energy_bias": -1.7833683490753174, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.9239981174468994, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9976933002471924, "metrics/particle/momentum/class_4/event/px_bias": 0.09230363368988037, "metrics/particle/momentum/class_4/event/py_bias": 0.012391604483127594, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7626945972442627, "metrics/particle/momentum/class_4/event/pz_bias": 0.4888347387313843, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.398190498352051, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016988692805171013, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1793087124824524, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1753106564283371, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.013704770244657993, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21396595239639282, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7241228818893433, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.06629887968301773, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.856815814971924, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.081778883934021, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.33169153332710266, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.30544307827949524, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.09231025725603104, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.12179487198591232, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6918899416923523, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.63674259185791, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05246124044060707, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.44086217880249, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8616352081298828, "metrics/particle/momentum/class_5/event/energy_bias": -1.5361459255218506, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7661967277526855, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.103825092315674, "metrics/particle/momentum/class_5/event/px_bias": 0.013571083545684814, "metrics/particle/momentum/class_5/event/py_bias": 0.4483526647090912, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4394384622573853, "metrics/particle/momentum/class_5/event/pz_bias": -0.08993729948997498, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.668793678283691, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4043384790420532, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4051835536956787, "metrics/set_slot/angular_recall_all_dr0p10": 0.9461368322372437, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8116183280944824, "metrics/set_slot/soft_count_bias_mean": 0.6451948881149292}, "step_train_time": 4.938815116882324, "step_valid_time": 22232.48335003853, "step_total_time": 22237.42216539383, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0068392173367857, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1741469635774191, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3011, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2838, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2800, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9375619425173439, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9425440053138492, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9400463729711825, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05745599468615078, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9250082590023125, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9299236134174693, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9274594236502153, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9375619425173439, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976549045041061, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16427157951877758, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2112, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372236958443855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9434801958166444, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9403415391439344, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05651980418335559, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9336870026525199, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9399198931909212, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9367930805056554, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372236958443855, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021789799202758, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1607667831879117, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1270, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1178, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9145962732919255, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9275590551181102, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9210320562939797, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07244094488188976, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9114906832298136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9244094488188976, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9179046129788898, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9145962732919255} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_45000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..9a99f38286be4ecac07899b9f8171fdd34fda075 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131893121091529, "train": {"Classification_binary": 0.20709997415542603, "Cardinality": 0.14874520897865295, "Classification": 0.09464523941278458, "Regression_pt": 0.1808646023273468, "Regression_eta": 0.0017907429719343781, "Regression_sin_phi": 0.0010215468937531114, "Regression_cos_phi": 0.0009268899448215961, "Regression_energy": 0.1805732399225235, "HitGrouping": 0.23837804794311523, "Auxiliary": 0.2854068875312805, "Total": 1.3237900733947754, "Total_unweighted": 0.66692223562859}, "train_group": {}, "valid": {"Classification_binary": 0.18406963348388672, "Cardinality": 0.12912357846895853, "Classification": 0.09040601054827373, "Regression_pt": 0.1725369095802307, "Regression_eta": 0.0016632831345001857, "Regression_sin_phi": 0.0009379583255698284, "Regression_cos_phi": 0.000953133994092544, "Regression_energy": 0.17210900783538818, "HitGrouping": 0.23306238651275635, "Auxiliary": 0.2688764731089274, "Total": 1.2398085594177246, "Total_unweighted": 0.6226759369019419, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7362839579582214, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.35432833433151245, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43411731719970703, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.345062792301178, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.35266679525375366, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6019646525382996, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6859075427055359, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3035320043563843, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.40562522411346436, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.29094576835632324, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3310980796813965, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6740087866783142, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7539109587669373, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3540230393409729, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.41343870759010315, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.34520652890205383, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34088069200515747, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5677099227905273, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7421390414237976, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3495924472808838, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41939210891723633, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3402932584285736, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3437959551811218, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5882675051689148, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025764334946870804, "metrics/hit_grouping/calorimeter/background_recall": 0.6319887042045593, "metrics/hit_grouping/calorimeter/completeness": 0.3972609341144562, "metrics/hit_grouping/calorimeter/energy_completeness": 0.422708660364151, "metrics/hit_grouping/calorimeter/energy_purity": 0.41775381565093994, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4942612648010254, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7626500129699707, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3665286898612976, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3502538204193115, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.35982587933540344, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3673951327800751, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5377127528190613, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7385707497596741, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.36087295413017273, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.46021348237991333, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3490780293941498, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.35345524549484253, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6012857556343079, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7024425268173218, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3019886612892151, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5231955647468567, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2897946536540985, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2837781608104706, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6817089915275574, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7265573740005493, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3727976977825165, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5075677633285522, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3623465299606323, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3624827265739441, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6280909180641174, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7222736477851868, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.44379913806915283, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6253270506858826, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.429440975189209, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.42598649859428406, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6232339143753052, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7614323496818542, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3125903606414795, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2982456088066101, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3051770329475403, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3131890594959259, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.540980339050293, "metrics/hit_grouping/calorimeter/purity": 0.39139989018440247, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8420146107673645, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07122350484132767, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1599999964237213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07271428406238556, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.032893892377614975, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05846153944730759, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.755269467830658, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14343605935573578, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10547320544719696, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14247596263885498, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14550146460533142, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.42246294021606445, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7392550706863403, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3673829138278961, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.44297611713409424, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3573741614818573, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3618276119232178, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6090962886810303, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7155672907829285, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4385833740234375, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4425588846206665, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45925331115722656, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6189479827880859, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7776413559913635, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.33426809310913086, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4802137613296509, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47873377799987793, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4655075669288635, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6743496060371399, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44372686743736267, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39781564474105835, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4040018320083618, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6765377521514893, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6960762739181519, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43288305401802063, "metrics/hit_grouping/tracker/all/object_completeness": 0.4188971519470215, "metrics/hit_grouping/tracker/all/object_purity": 0.4277591407299042, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6405630111694336, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005554250907152891, "metrics/hit_grouping/tracker/background_recall": 0.027424035593867302, "metrics/hit_grouping/tracker/completeness": 0.436380535364151, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5107202529907227, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7059189677238464, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4246924817562103, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4718022048473358, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.48505333065986633, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6359909176826477, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6872757077217102, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4553588330745697, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4024547338485718, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.41205504536628723, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6527698636054993, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6835775971412659, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42718446254730225, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3189254403114319, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31725963950157166, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6377426981925964, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7185588479042053, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45875269174575806, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43955129384994507, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4417959451675415, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6069374680519104, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7473140358924866, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44540613889694214, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.447079598903656, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4389696717262268, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5352863073348999, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6577780246734619, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3969796299934387, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38703465461730957, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4075016677379608, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7028799057006836, "metrics/hit_grouping/tracker/purity": 0.41622090339660645, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8857017755508423, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018679408356547356, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.031848348677158356, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03416739031672478, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08253692090511322, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6706682443618774, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2218766212463379, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22282597422599792, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22534674406051636, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5795748829841614, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6732024550437927, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5131928324699402, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49381643533706665, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5042895674705505, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7233338952064514, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.40722203254699707, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5451146364212036, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.600709080696106, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6093020439147949, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07850527763366699, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07849729061126709, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.085893630981445, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3017515242099762, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17131169140338898, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8502770066261292, "metrics/particle/class_1/pid_accuracy": 0.963767945766449, "metrics/particle/class_2/efficiency": 0.40083014965057373, "metrics/particle/class_2/pid_accuracy": 0.5883011221885681, "metrics/particle/class_3/efficiency": 0.7075064778327942, "metrics/particle/class_3/pid_accuracy": 0.9603686928749084, "metrics/particle/class_4/efficiency": 0.6862367391586304, "metrics/particle/class_4/pid_accuracy": 0.36465638875961304, "metrics/particle/class_5/efficiency": 0.8659999966621399, "metrics/particle/class_5/pid_accuracy": 0.4249422550201416, "metrics/particle/count/bias_mean": -3.1184895038604736, "metrics/particle/count/mae": 4.537760257720947, "metrics/particle/count/prediction_mean": 66.9375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07618057727813721, "metrics/particle/event/energy_response_mean": 0.9681589007377625, "metrics/particle/event/met_abs_error": 8.486149787902832, "metrics/particle/event/scalar_pt_relative_abs_error": 0.062012672424316406, "metrics/particle/event/scalar_pt_response_mean": 0.9855990409851074, "metrics/particle/event/vector_pt_closure": 0.06472154706716537, "metrics/particle/matched/delta_eta_abs_mean": 0.014424800872802734, "metrics/particle/matched/delta_phi_abs_mean": 0.014765721745789051, "metrics/particle/matched/delta_r_mean": 0.023102356120944023, "metrics/particle/matched/energy_relative_abs_mean": 0.1644344925880432, "metrics/particle/matched/pid_accuracy": 0.9375612139701843, "metrics/particle/matched/pt_relative_abs_mean": 0.1639823317527771, "metrics/particle/matching/duplicate_fraction": 0.06339479982852936, "metrics/particle/matching/efficiency": 0.7496886849403381, "metrics/particle/matching/f1": 0.7667544484138489, "metrics/particle/matching/purity": 0.7846152186393738, "metrics/particle/matching/target_coverage_dr0p05": 0.7138821482658386, "metrics/particle/matching/target_coverage_dr0p10": 0.8178168535232544, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5149527192115784, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7887757420539856, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02221553772687912, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16185085475444794, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1618826985359192, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011511163786053658, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2025635689496994, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008867241442203522, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.830352246761322, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03919572755694389, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.167769432067871, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8007.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28919002413749695, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1988266706466675, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2519465684890747, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.06999671459198, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16397160291671753, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.82043170928955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019226977601647377, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2535234689712524, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12243178486824036, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3564982414245605, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10142.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.732168972492218, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.8841005563735962, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.552433967590332, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.311363220214844, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2173631191253662, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2222173511981964, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.144766807556152, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1947451829910278, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.261557579040527, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022855378687381744, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16596533358097076, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16712498664855957, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01758611761033535, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20926593244075775, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.036934949457645416, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7336772680282593, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09726830571889877, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.230778217315674, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1814.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5285590887069702, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3612862825393677, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.0529656410217285, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.864959716796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2106943279504776, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 27.492610931396484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06983239948749542, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8784492015838623, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1273506134748459, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.719940185546875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2506.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6004634499549866, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.033237457275391, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.579051971435547, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4782658815383911, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07666793465614319, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04241430386900902, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.39525032043457, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.0854802131652832, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.313980102539062, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02328508161008358, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16502389311790466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16487199068069458, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015963558107614517, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2057979702949524, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007188642048276961, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.28345000743865967, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.012662474997341633, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8412159085273743, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16693.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22307033836841583, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.464849591255188, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1637436151504517, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9805996417999268, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15762834250926971, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.49194622039795, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0052542779594659805, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5088308453559875, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01740805245935917, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.217034101486206, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20745.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7787003517150879, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5097953677177429, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.494879722595215, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.940191268920898, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.340576171875, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.4975166916847229, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.631789207458496, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.43444201350212097, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.64439582824707, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022932689636945724, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1641300618648529, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16412338614463806, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014729985035955906, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20506684482097626, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005657388363033533, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5478295087814331, "metrics/particle/momentum/all/accepted/pz_bias": 0.026463737711310387, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7576335668563843, "metrics/particle/momentum/all/accepted/samples": 26514.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26607757806777954, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3762820959091187, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3323101997375488, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1491639614105225, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16353726387023926, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.963685989379883, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014344323426485062, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9128358364105225, "metrics/particle/momentum/all/assigned/pz_bias": 0.038441967219114304, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.30873966217041, "metrics/particle/momentum/all/assigned/samples": 33393.0, "metrics/particle/momentum/all/efficiency": 0.7491100430488586, "metrics/particle/momentum/all/event/energy_bias": -7.407542705535889, "metrics/particle/momentum/all/event/p3_error_mean": 21.901710510253906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.431278228759766, "metrics/particle/momentum/all/event/px_bias": 1.6346070766448975, "metrics/particle/momentum/all/event/py_bias": -0.7621483206748962, "metrics/particle/momentum/all/event/pz_abs_mean": 15.843950271606445, "metrics/particle/momentum/all/event/pz_bias": 1.5437068939208984, "metrics/particle/momentum/all/event/pz_residual_rms": 25.406509399414062, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024486154317855835, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1600673794746399, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16019004583358765, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01946236565709114, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20075377821922302, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005771396216005087, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5849720239639282, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.033619120717048645, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7418166399002075, "metrics/particle/momentum/class_1/accepted/samples": 14208.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12180731445550919, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.413985937833786, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4434400200843811, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.285288542509079, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10212087631225586, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.3202757835388184, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010297903791069984, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7804948687553406, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.053757503628730774, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7299938201904297, "metrics/particle/momentum/class_1/assigned/samples": 16314.0, "metrics/particle/momentum/class_1/efficiency": 0.8514412641525269, "metrics/particle/momentum/class_1/event/energy_bias": 8.325586318969727, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.294498443603516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.031840324401855, "metrics/particle/momentum/class_1/event/px_bias": 0.6592060923576355, "metrics/particle/momentum/class_1/event/py_bias": -0.45769357681274414, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.075299263000488, "metrics/particle/momentum/class_1/event/pz_bias": 1.068813681602478, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.41396713256836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033764809370040894, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20647114515304565, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20767563581466675, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03348696976900101, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24821285903453827, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03299492225050926, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9549317359924316, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.15570983290672302, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.420498847961426, "metrics/particle/momentum/class_2/accepted/samples": 1182.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7393553853034973, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8386759757995605, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0793043375015259, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6495272517204285, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3348644971847534, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.282536029815674, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05925585702061653, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.821655035018921, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08596903830766678, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.595627784729004, "metrics/particle/momentum/class_2/assigned/samples": 2177.0, "metrics/particle/momentum/class_2/efficiency": 0.39704400300979614, "metrics/particle/momentum/class_2/event/energy_bias": -16.966983795166016, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.966440200805664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.25549602508545, "metrics/particle/momentum/class_2/event/px_bias": 1.4071980714797974, "metrics/particle/momentum/class_2/event/py_bias": -0.5904098749160767, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.551665306091309, "metrics/particle/momentum/class_2/event/pz_bias": 1.2122292518615723, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.96764373779297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019711529836058617, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16402168571949005, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16384395956993103, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013132059015333652, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.204759418964386, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002049944130703807, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3076092302799225, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0037594125606119633, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0345025062561035, "metrics/particle/momentum/class_3/accepted/samples": 10732.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3581172823905945, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5351898670196533, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3626577854156494, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.1875853538513184, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20638889074325562, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.203441619873047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011666666716337204, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7447434663772583, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008742449805140495, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.114450693130493, "metrics/particle/momentum/class_3/assigned/samples": 14400.0, "metrics/particle/momentum/class_3/efficiency": 0.7057276368141174, "metrics/particle/momentum/class_3/event/energy_bias": 3.547412633895874, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.94542407989502, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.837613105773926, "metrics/particle/momentum/class_3/event/px_bias": -0.4420250654220581, "metrics/particle/momentum/class_3/event/py_bias": -0.08400984108448029, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.30831241607666, "metrics/particle/momentum/class_3/event/pz_bias": -0.8971492648124695, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.976404190063477, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024888960644602776, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17700588703155518, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17170700430870056, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05244727432727814, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21115557849407196, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01171875, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.134106993675232, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4170280992984772, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6741113662719727, "metrics/particle/momentum/class_4/accepted/samples": 256.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.361569344997406, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4011130332946777, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.434623956680298, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.311462163925171, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2225433588027954, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.277331352233887, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014450866729021072, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.315775752067566, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.45793747901916504, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.701188564300537, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.7032967209815979, "metrics/particle/momentum/class_4/event/energy_bias": -1.1513170003890991, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.922027587890625, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.042419672012329, "metrics/particle/momentum/class_4/event/px_bias": 0.07343389093875885, "metrics/particle/momentum/class_4/event/py_bias": 0.05292163044214249, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.754063367843628, "metrics/particle/momentum/class_4/event/pz_bias": 0.31666791439056396, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.128848552703857, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017002301290631294, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.204884335398674, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20429782569408417, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.005504876375198364, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24516841769218445, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029411764815449715, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2907822132110596, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.7879060506820679, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.191948413848877, "metrics/particle/momentum/class_5/accepted/samples": 136.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04100307822227478, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.26347479224205017, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2644440531730652, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0308871790766716, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.10897435992956161, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3495274484157562, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.7373454570770264, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4153822958469391, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.587859153747559, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8553459048271179, "metrics/particle/momentum/class_5/event/energy_bias": -1.16224205493927, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0231497287750244, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2438368797302246, "metrics/particle/momentum/class_5/event/px_bias": -0.06320594251155853, "metrics/particle/momentum/class_5/event/py_bias": 0.31704339385032654, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6705400943756104, "metrics/particle/momentum/class_5/event/pz_bias": -0.15685461461544037, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.1475372314453125, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.40727901458740234, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.40919435024261475, "metrics/set_slot/angular_recall_all_dr0p10": 0.9455978274345398, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8096667528152466, "metrics/set_slot/soft_count_bias_mean": 0.19183774292469025}, "step_train_time": 5.701291799545288, "step_valid_time": 24760.7099070549, "step_total_time": 24766.411199092865, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005819539378125, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16159266525705207, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3004, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2841, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2802, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385530227948464, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9457390146471372, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9421323163654453, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05426098535286285, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9256689791873142, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9327563249001332, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9291991377880948, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385530227948464, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9881966526245868, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16354470293925616, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2124, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9389920424403183, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9431616341030196, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9410722197607444, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056838365896980464, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9350132625994695, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9391651865008881, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9370846256092158, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9389920424403183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9886688647450349, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15273402708188688, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1274, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1172, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1164, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9099378881987578, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9199372056514914, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9149102263856362, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08006279434850863, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9037267080745341, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9136577708006279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9086651053864169, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9099378881987578} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_46000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..19128685f33121562471cfe8a4234812c95f18a6 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.179268523782452, "train": {"Classification_binary": 0.1936531364917755, "Cardinality": 0.1378796249628067, "Classification": 0.0791851133108139, "Regression_pt": 0.16933207213878632, "Regression_eta": 0.0014669247902929783, "Regression_sin_phi": 0.0007306779152713716, "Regression_cos_phi": 0.0008351951837539673, "Regression_energy": 0.16812770068645477, "HitGrouping": 0.2251061201095581, "Auxiliary": 0.274892657995224, "Total": 1.214453101158142, "Total_unweighted": 0.6133308205171488}, "train_group": {}, "valid": {"Classification_binary": 0.18425538142522177, "Cardinality": 0.12713146209716797, "Classification": 0.08667108416557312, "Regression_pt": 0.16680238644282022, "Regression_eta": 0.0016515996928016345, "Regression_sin_phi": 0.0009147766201446453, "Regression_cos_phi": 0.0009452746404955784, "Regression_energy": 0.16611754894256592, "HitGrouping": 0.22839524348576865, "Auxiliary": 0.26415034135182697, "Total": 1.2178625265757244, "Total_unweighted": 0.6073580519296229, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.750677227973938, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.36981162428855896, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42419949173927307, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.36429768800735474, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3665546476840973, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.565977156162262, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6982349157333374, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.33282724022865295, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4073195457458496, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3267103433609009, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3614477217197418, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6299559473991394, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7658826112747192, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36766719818115234, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.39990702271461487, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.36144396662712097, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.35458865761756897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5370312929153442, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7549219131469727, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3652322292327881, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.40838906168937683, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35924679040908813, "metrics/hit_grouping/calorimeter/all/object_purity": 0.35905444622039795, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5546808242797852, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02597084268927574, "metrics/hit_grouping/calorimeter/background_recall": 0.6343452334403992, "metrics/hit_grouping/calorimeter/completeness": 0.41793832182884216, "metrics/hit_grouping/calorimeter/energy_completeness": 0.437143474817276, "metrics/hit_grouping/calorimeter/energy_purity": 0.4320412278175354, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.47712281346321106, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7767471075057983, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.381207674741745, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3361600339412689, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37842798233032227, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.38200873136520386, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5015228390693665, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.751116931438446, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3787962794303894, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.44820475578308105, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36972853541374207, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3719111979007721, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5696263909339905, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7126902937889099, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3172825276851654, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5197826027870178, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.30772367119789124, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29706987738609314, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.651624321937561, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.740979790687561, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.39145955443382263, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49461227655410767, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3844190537929535, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.38465794920921326, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5917159914970398, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7350828647613525, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.45923057198524475, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6138147711753845, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.44913220405578613, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4526215195655823, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5960230231285095, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.772449254989624, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3250078558921814, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2894067168235779, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.32068300247192383, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.31955593824386597, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5095754861831665, "metrics/hit_grouping/calorimeter/purity": 0.41185951232910156, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8444920778274536, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07147932052612305, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.18000000715255737, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07405859977006912, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.036333199590444565, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05538461357355118, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7499374747276306, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14145690202713013, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1145952120423317, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14145322144031525, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13710615038871765, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4053591787815094, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7532994151115417, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3843640089035034, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4301547706127167, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3778519630432129, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37869852781295776, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5739113092422485, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7205048203468323, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.437836229801178, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.44192275404930115, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45584970712661743, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6217871904373169, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7719669938087463, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32809871435165405, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4699589014053345, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4807482063770294, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48569825291633606, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6714708805084229, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4404977560043335, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3930271565914154, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4012022018432617, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6837835907936096, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6954326629638672, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4301908314228058, "metrics/hit_grouping/tracker/all/object_completeness": 0.4149608016014099, "metrics/hit_grouping/tracker/all/object_purity": 0.4251740574836731, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6475061178207397, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004803923889994621, "metrics/hit_grouping/tracker/background_recall": 0.023396315053105354, "metrics/hit_grouping/tracker/completeness": 0.4345797300338745, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5165495872497559, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7049742341041565, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4205922484397888, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46750470995903015, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4848743975162506, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6431890726089478, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6876453161239624, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45231297612190247, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.39729589223861694, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.410329133272171, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6584808826446533, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6825257539749146, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4279935359954834, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3170894980430603, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3084201514720917, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6454287767410278, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.716768741607666, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4502156674861908, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43547624349594116, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.43828085064888, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6160136461257935, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7474485635757446, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4347536563873291, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4421500563621521, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4261312186717987, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5326231718063354, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6584910750389099, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40330132842063904, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3834536075592041, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40793102979660034, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.708733320236206, "metrics/hit_grouping/tracker/purity": 0.41419079899787903, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8860735297203064, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018679408356547356, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.029707539826631546, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.030509689822793007, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07688966393470764, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6754830479621887, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21669258177280426, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22254863381385803, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2280583381652832, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5806117057800293, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.67179936170578, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5104185342788696, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4892194867134094, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5012471079826355, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7326604127883911, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4028099477291107, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5725940465927124, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6349220275878906, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5959326028823853, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08291736990213394, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06512782722711563, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.135150909423828, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30009201169013977, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16965216398239136, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8589631915092468, "metrics/particle/class_1/pid_accuracy": 0.9630056023597717, "metrics/particle/class_2/efficiency": 0.4104779064655304, "metrics/particle/class_2/pid_accuracy": 0.6299535632133484, "metrics/particle/class_3/efficiency": 0.7238752841949463, "metrics/particle/class_3/pid_accuracy": 0.9620651006698608, "metrics/particle/class_4/efficiency": 0.6862367391586304, "metrics/particle/class_4/pid_accuracy": 0.3814866840839386, "metrics/particle/class_5/efficiency": 0.8700000047683716, "metrics/particle/class_5/pid_accuracy": 0.3931034505367279, "metrics/particle/count/bias_mean": -2.5657551288604736, "metrics/particle/count/mae": 4.244140625, "metrics/particle/count/prediction_mean": 67.490234375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07508669048547745, "metrics/particle/event/energy_response_mean": 0.966661274433136, "metrics/particle/event/met_abs_error": 7.7273850440979, "metrics/particle/event/scalar_pt_relative_abs_error": 0.061660345643758774, "metrics/particle/event/scalar_pt_response_mean": 0.9777719378471375, "metrics/particle/event/vector_pt_closure": 0.0626305490732193, "metrics/particle/matched/delta_eta_abs_mean": 0.014315118081867695, "metrics/particle/matched/delta_phi_abs_mean": 0.01454939879477024, "metrics/particle/matched/delta_r_mean": 0.022887805476784706, "metrics/particle/matched/energy_relative_abs_mean": 0.16152948141098022, "metrics/particle/matched/pid_accuracy": 0.9396664500236511, "metrics/particle/matched/pt_relative_abs_mean": 0.16046790778636932, "metrics/particle/matching/duplicate_fraction": 0.06444798409938812, "metrics/particle/matching/efficiency": 0.7616768479347229, "metrics/particle/matching/f1": 0.7758849859237671, "metrics/particle/matching/purity": 0.7906332612037659, "metrics/particle/matching/target_coverage_dr0p05": 0.7231009602546692, "metrics/particle/matching/target_coverage_dr0p10": 0.8236250877380371, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5327398180961609, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8009218573570251, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022117046639323235, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15968340635299683, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15857896208763123, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006169883534312248, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19904233515262604, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009834050200879574, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8375651240348816, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.017592795193195343, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.264913320541382, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8135.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2902349531650543, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1112706661224365, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2494876384735107, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0621349811553955, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1582818478345871, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.266328811645508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020202020183205605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2490339279174805, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014404760673642159, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.437438488006592, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10197.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7438734173774719, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.6592899560928345, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.227823257446289, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.232007026672363, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13507188856601715, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.25036436319351196, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.857067108154297, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7377610802650452, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.523433685302734, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0226422231644392, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1605716496706009, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16010473668575287, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021584611386060715, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20252791047096252, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.031301483511924744, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.587741732597351, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.05117809399962425, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.942446231842041, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1821.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5128045082092285, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1115670204162598, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.048304557800293, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8679916858673096, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20725388824939728, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 13.110905647277832, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06536468863487244, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8080594539642334, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.12563854455947876, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.644964218139648, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2509.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6027805209159851, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.925365447998047, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.232309341430664, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3497660160064697, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0038616303354501724, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.002864271402359009, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.065547943115234, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6728432178497314, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.69086265563965, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0232855211943388, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16215991973876953, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16076691448688507, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0006488809594884515, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2008996307849884, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008820935036055744, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2748279869556427, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013703887350857258, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.823235034942627, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17005.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21906857192516327, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2434717416763306, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0020750761032104, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8079060316085815, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15225806832313538, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.428101539611816, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005256869830191135, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4890458881855011, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005840774159878492, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8857643604278564, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20925.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7932546734809875, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.2280333042144775, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.92402458190918, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.262645721435547, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1866155862808228, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8679358959197998, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.612072467803955, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5472851991653442, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.617383003234863, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022889506071805954, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16130541265010834, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1600620001554489, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0037287860177457333, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20045173168182373, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005637772846966982, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5333006978034973, "metrics/particle/momentum/all/accepted/pz_bias": -0.00012158755271229893, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7392772436141968, "metrics/particle/momentum/all/accepted/samples": 26961.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2625601887702942, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1935474872589111, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1551436185836792, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9640753269195557, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1581873893737793, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.431709289550781, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014272546395659447, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8924832940101624, "metrics/particle/momentum/all/assigned/pz_bias": 0.017374763265252113, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3720569610595703, "metrics/particle/momentum/all/assigned/samples": 33631.0, "metrics/particle/momentum/all/efficiency": 0.7617392539978027, "metrics/particle/momentum/all/event/energy_bias": -8.812688827514648, "metrics/particle/momentum/all/event/p3_error_mean": 21.24689483642578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.713603019714355, "metrics/particle/momentum/all/event/px_bias": 1.0554051399230957, "metrics/particle/momentum/all/event/py_bias": -1.11543607711792, "metrics/particle/momentum/all/event/pz_abs_mean": 15.684797286987305, "metrics/particle/momentum/all/event/pz_bias": -0.3877970576286316, "metrics/particle/momentum/all/event/pz_residual_rms": 25.257144927978516, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02423647791147232, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15427178144454956, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1543770134449005, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006230434402823448, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1940091848373413, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0054272194392979145, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5785490274429321, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0015045112231746316, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7517527341842651, "metrics/particle/momentum/class_1/accepted/samples": 14372.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1222534030675888, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3446865975856781, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3663899898529053, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19795018434524536, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0929720476269722, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.475205421447754, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009174873121082783, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7422177791595459, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.026759255677461624, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6862220764160156, "metrics/particle/momentum/class_1/assigned/samples": 16349.0, "metrics/particle/momentum/class_1/efficiency": 0.8612692356109619, "metrics/particle/momentum/class_1/event/energy_bias": 2.586303472518921, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.586246490478516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.31053352355957, "metrics/particle/momentum/class_1/event/px_bias": 0.3658708333969116, "metrics/particle/momentum/class_1/event/py_bias": 0.621741771697998, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.686458587646484, "metrics/particle/momentum/class_1/event/pz_bias": -1.0063937902450562, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.785545349121094, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03287089988589287, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2019948661327362, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20192919671535492, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.028027407824993134, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2409033477306366, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0387788787484169, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.867189884185791, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.12689170241355896, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.352195739746094, "metrics/particle/momentum/class_2/accepted/samples": 1212.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7901210784912109, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9707657694816589, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.054815649986267, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6271383762359619, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35312360525131226, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.361982822418213, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06645990163087845, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.989286184310913, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.021505096927285194, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.281785488128662, "metrics/particle/momentum/class_2/assigned/samples": 2257.0, "metrics/particle/momentum/class_2/efficiency": 0.4071212708950043, "metrics/particle/momentum/class_2/event/energy_bias": -11.474534034729004, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.44229507446289, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.790209770202637, "metrics/particle/momentum/class_2/event/px_bias": 1.7807537317276, "metrics/particle/momentum/class_2/event/py_bias": -2.1182031631469727, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.4689302444458, "metrics/particle/momentum/class_2/event/pz_bias": 0.11043715476989746, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.402240753173828, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020084960386157036, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16548307240009308, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1624818593263626, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.003425673581659794, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20334479212760925, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017296313308179379, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2975202798843384, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.016308343037962914, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0119160413742065, "metrics/particle/momentum/class_3/accepted/samples": 10985.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3392840623855591, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.131577730178833, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.04018497467041, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8591912984848022, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20008264482021332, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.51348304748535, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011226668953895569, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7077591419219971, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.012670854106545448, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0472755432128906, "metrics/particle/momentum/class_3/assigned/samples": 14519.0, "metrics/particle/momentum/class_3/efficiency": 0.7223647236824036, "metrics/particle/momentum/class_3/event/energy_bias": 2.8952841758728027, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.758315086364746, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.537248611450195, "metrics/particle/momentum/class_3/event/px_bias": -0.9988125562667847, "metrics/particle/momentum/class_3/event/py_bias": 0.31377139687538147, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.292411804199219, "metrics/particle/momentum/class_3/event/pz_bias": 0.31374403834342957, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.342794418334961, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02420317754149437, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17186369001865387, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16665655374526978, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.042405664920806885, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20884695649147034, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.015625, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.023040771484375, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3559090495109558, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2565832138061523, "metrics/particle/momentum/class_4/accepted/samples": 256.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.31648439168930054, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.7161059379577637, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.261364459991455, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.131399631500244, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2314285784959793, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.863481521606445, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022857142612338066, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.328917384147644, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.23664654791355133, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.1517653465271, "metrics/particle/momentum/class_4/assigned/samples": 350.0, "metrics/particle/momentum/class_4/efficiency": 0.7032967209815979, "metrics/particle/momentum/class_4/event/energy_bias": -1.1319433450698853, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.10943865776062, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1223843097686768, "metrics/particle/momentum/class_4/event/px_bias": -0.014870941638946533, "metrics/particle/momentum/class_4/event/py_bias": -0.0956350788474083, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8613985776901245, "metrics/particle/momentum/class_4/event/pz_bias": 0.4176406264305115, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.596024036407471, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015650248154997826, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.184665709733963, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17985352873802185, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.025953389704227448, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22092343866825104, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029411764815449715, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9869052171707153, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6666120290756226, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.7843449115753174, "metrics/particle/momentum/class_5/accepted/samples": 136.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07245068997144699, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.41591620445251465, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4159487187862396, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.201854407787323, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.10897435992956161, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.2359695434570312, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.51725172996521, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.45778146386146545, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.005414962768555, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8553459048271179, "metrics/particle/momentum/class_5/event/energy_bias": -1.687800645828247, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.8261194229125977, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.087555408477783, "metrics/particle/momentum/class_5/event/px_bias": -0.0775357037782669, "metrics/particle/momentum/class_5/event/py_bias": 0.16288897395133972, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5390352010726929, "metrics/particle/momentum/class_5/event/pz_bias": -0.22322535514831543, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.820338249206543, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.40038684010505676, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.40122318267822266, "metrics/set_slot/angular_recall_all_dr0p10": 0.9488318562507629, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8146200180053711, "metrics/set_slot/soft_count_bias_mean": 0.598176896572113}, "step_train_time": 5.030275821685791, "step_valid_time": 27291.476471185684, "step_total_time": 27296.506747722626, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.988051750702391, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16518771898550044, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2835, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2804, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9365708622398414, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.945, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9407665505226481, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9263296993723158, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346666666666666, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9304795088767214, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9365708622398414, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9809800268818826, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15702130545915027, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2108, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2099, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9319186560565871, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9406514948683623, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9362647124139463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05934850513163766, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9279398762157383, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9366354306113343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9322673773040195, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9319186560565871, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9850657185609853, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1477105642996206, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1189, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1180, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9231366459627329, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9318181818181818, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9274570982839313, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06818181818181818, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9161490683229814, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9247648902821317, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9204368174726989, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9231366459627329} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_47000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a661654a2a69139d78871ade2f5da98694c92c7 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226643926473375, "train": {"Classification_binary": 0.19460411369800568, "Cardinality": 0.13740186393260956, "Classification": 0.08862239122390747, "Regression_pt": 0.1645946502685547, "Regression_eta": 0.0017661318415775895, "Regression_sin_phi": 0.0009949117666110396, "Regression_cos_phi": 0.0008956253877840936, "Regression_energy": 0.16286969184875488, "HitGrouping": 0.2292504757642746, "Auxiliary": 0.27610936760902405, "Total": 1.2559874057769775, "Total_unweighted": 0.6143475160351954}, "train_group": {}, "valid": {"Classification_binary": 0.17922506729761759, "Cardinality": 0.12768828868865967, "Classification": 0.08974586923917134, "Regression_pt": 0.16270663340886435, "Regression_eta": 0.001580366554359595, "Regression_sin_phi": 0.0008741041334966818, "Regression_cos_phi": 0.0008897723164409399, "Regression_energy": 0.16229114929835, "HitGrouping": 0.22765556971232095, "Auxiliary": 0.2612553834915161, "Total": 1.1978657245635986, "Total_unweighted": 0.5973129622483005, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7477231025695801, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.365876168012619, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.420987993478775, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.35776209831237793, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3607071042060852, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5713611245155334, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6904681324958801, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3140098452568054, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39952558279037476, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3065902590751648, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3419272303581238, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6496102809906006, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7639592885971069, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36419713497161865, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4027993083000183, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35773175954818726, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3547741174697876, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5410597920417786, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.752142608165741, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3602358400821686, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4083586633205414, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35315439105033875, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3555310070514679, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5605471134185791, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025952767580747604, "metrics/hit_grouping/calorimeter/background_recall": 0.6337653398513794, "metrics/hit_grouping/calorimeter/completeness": 0.4121888279914856, "metrics/hit_grouping/calorimeter/energy_completeness": 0.43461018800735474, "metrics/hit_grouping/calorimeter/energy_purity": 0.42954838275909424, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.47966933250427246, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.769672691822052, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.376053124666214, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3376530408859253, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37112346291542053, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37904202938079834, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5166915655136108, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7498171925544739, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3759804368019104, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4534206688404083, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36655086278915405, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.367476224899292, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5663512945175171, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7174606323242188, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3103484809398651, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5110605359077454, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3011592924594879, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29331785440444946, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6473265290260315, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7411225438117981, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.38435226678848267, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49878543615341187, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3770865499973297, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3758743107318878, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5913422703742981, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.745259165763855, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.46294113993644714, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6132914423942566, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4503830075263977, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4452042281627655, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5761381387710571, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7648707628250122, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32116663455963135, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2849203050136566, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3149840831756592, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.32218578457832336, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.525445282459259, "metrics/hit_grouping/calorimeter/purity": 0.4061861038208008, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.841196596622467, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07200554758310318, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16769230365753174, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07239926606416702, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04142766445875168, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05846153944730759, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7426145076751709, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1511194109916687, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11288483440876007, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14921557903289795, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1551014631986618, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4064994156360626, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7507925629615784, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3784067630767822, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4304826855659485, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.37086814641952515, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37375375628471375, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5801088809967041, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7171016335487366, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4346981346607208, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.44031310081481934, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4584944248199463, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6276150345802307, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7743320465087891, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3135165572166443, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46893593668937683, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48958897590637207, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48289400339126587, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6754559278488159, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44112783670425415, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3976321816444397, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40133264660835266, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6726785898208618, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6969460844993591, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42834874987602234, "metrics/hit_grouping/tracker/all/object_completeness": 0.4171275794506073, "metrics/hit_grouping/tracker/all/object_purity": 0.4268327057361603, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6424522995948792, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006691941991448402, "metrics/hit_grouping/tracker/background_recall": 0.04009950906038284, "metrics/hit_grouping/tracker/completeness": 0.43505245447158813, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5156359076499939, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7057338953018188, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41758543252944946, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4682503640651703, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.48338770866394043, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6388154625892639, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6916436553001404, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44736340641975403, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4018232226371765, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4078368842601776, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6520083546638489, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6830722093582153, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4320388436317444, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3199027478694916, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3214713931083679, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6403721570968628, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7206052541732788, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45129403471946716, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4369348883628845, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.43944698572158813, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6072070598602295, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7414092421531677, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.444740355014801, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45165303349494934, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4467811584472656, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5632489919662476, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6583059430122375, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39557480812072754, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38525286316871643, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40689197182655334, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7022945284843445, "metrics/hit_grouping/tracker/purity": 0.4154294431209564, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8789936900138855, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013900956138968468, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.026792895048856735, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.031159309670329094, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08253692090511322, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6653192043304443, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20839813351631165, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21211624145507812, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22817181050777435, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5987558364868164, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6758100390434265, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5097101926803589, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4935113191604614, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5032188296318054, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7235109806060791, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3988116383552551, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5695968866348267, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6330559849739075, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5899327993392944, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08691572397947311, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.059128064662218094, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.145906448364258, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2858763039112091, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15543648600578308, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8672932386398315, "metrics/particle/class_1/pid_accuracy": 0.9644331932067871, "metrics/particle/class_2/efficiency": 0.41653579473495483, "metrics/particle/class_2/pid_accuracy": 0.6054403185844421, "metrics/particle/class_3/efficiency": 0.7246047258377075, "metrics/particle/class_3/pid_accuracy": 0.9640277028083801, "metrics/particle/class_4/efficiency": 0.7093359231948853, "metrics/particle/class_4/pid_accuracy": 0.3731343150138855, "metrics/particle/class_5/efficiency": 0.8600000143051147, "metrics/particle/class_5/pid_accuracy": 0.4116279184818268, "metrics/particle/count/bias_mean": -2.9329426288604736, "metrics/particle/count/mae": 4.389974117279053, "metrics/particle/count/prediction_mean": 67.123046875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.071991465985775, "metrics/particle/event/energy_response_mean": 0.9836788177490234, "metrics/particle/event/met_abs_error": 7.192127227783203, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0582914762198925, "metrics/particle/event/scalar_pt_response_mean": 0.9943292140960693, "metrics/particle/event/vector_pt_closure": 0.06027670577168465, "metrics/particle/matched/delta_eta_abs_mean": 0.01455241534858942, "metrics/particle/matched/delta_phi_abs_mean": 0.014274492859840393, "metrics/particle/matched/delta_r_mean": 0.022811835631728172, "metrics/particle/matched/energy_relative_abs_mean": 0.16029426455497742, "metrics/particle/matched/pid_accuracy": 0.9399434924125671, "metrics/particle/matched/pt_relative_abs_mean": 0.15905390679836273, "metrics/particle/matching/duplicate_fraction": 0.06321956217288971, "metrics/particle/matching/efficiency": 0.7665836215019226, "metrics/particle/matching/f1": 0.7829735279083252, "metrics/particle/matching/purity": 0.8000795245170593, "metrics/particle/matching/target_coverage_dr0p05": 0.7314741015434265, "metrics/particle/matching/target_coverage_dr0p10": 0.8276396989822388, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5386781692504883, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8025574684143066, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022039048373699188, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15895912051200867, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1575177162885666, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004041824955493212, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19833026826381683, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009205426089465618, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8121709823608398, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.007818009704351425, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1676554679870605, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8256.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2674384117126465, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2225066423416138, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2895978689193726, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1088364124298096, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15113423764705658, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.659056663513184, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019051359966397285, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2156895399093628, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05779504403471947, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.37304949760437, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10183.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7549378275871277, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5938643217086792, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.259590148925781, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8445866107940674, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.22281908988952637, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.341571182012558, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.02455472946167, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.08199943602085114, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.513798713684082, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022901887074112892, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1627958118915558, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15999352931976318, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010514541529119015, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20088297128677368, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02920871041715145, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.64509916305542, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2015012502670288, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.121816158294678, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1883.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.47322359681129456, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2705000638961792, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3964571952819824, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.215283155441284, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19896744191646576, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.62522315979004, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0651310533285141, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.833405017852783, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2936696410179138, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.850553512573242, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2518.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6233035326004028, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.927011251449585, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.336617469787598, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.201934576034546, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.039718158543109894, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09606064856052399, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.196335792541504, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.8221938610076904, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.16309928894043, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022920580580830574, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16048048436641693, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15925855934619904, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012864080257713795, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19967924058437347, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007038948824629188, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26470205187797546, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013281877152621746, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7839304208755493, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17048.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21432209014892578, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4114218950271606, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0989460945129395, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9200031757354736, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14562685787677765, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.753997802734375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005292023532092571, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4952448904514313, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.03380899876356125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1996266841888428, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20786.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7952605485916138, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8631763458251953, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.154361724853516, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.643953323364258, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5470944046974182, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6147116422653198, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.245044231414795, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.746424674987793, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.230391502380371, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022651586681604385, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16017885506153107, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15878081321716309, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010022254660725594, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19935432076454163, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005259866826236248, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5265622138977051, "metrics/particle/momentum/all/accepted/pz_bias": -0.019910648465156555, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.728857159614563, "metrics/particle/momentum/all/accepted/samples": 27187.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24994179606437683, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3433785438537598, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2544852495193481, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0748217105865479, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.151312455534935, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.669098854064941, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013975572772324085, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8901376128196716, "metrics/particle/momentum/all/assigned/pz_bias": -0.025493087247014046, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3312559127807617, "metrics/particle/momentum/all/assigned/samples": 33487.0, "metrics/particle/momentum/all/efficiency": 0.768124520778656, "metrics/particle/momentum/all/event/energy_bias": -3.6576988697052, "metrics/particle/momentum/all/event/p3_error_mean": 19.40501594543457, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.720866203308105, "metrics/particle/momentum/all/event/px_bias": -0.8096315860748291, "metrics/particle/momentum/all/event/py_bias": -0.8602222204208374, "metrics/particle/momentum/all/event/pz_abs_mean": 14.444564819335938, "metrics/particle/momentum/all/event/pz_bias": -2.650618076324463, "metrics/particle/momentum/all/event/pz_residual_rms": 23.06184959411621, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023938482627272606, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1523839831352234, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15191750228405, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011903935112059116, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1916540265083313, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005782336462289095, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5612696409225464, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.020101433619856834, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7088900804519653, "metrics/particle/momentum/class_1/accepted/samples": 14527.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11516334861516953, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3744804561138153, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.38784992694854736, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23155970871448517, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08625122159719467, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.7497782707214355, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009488246403634548, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.717867374420166, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0023832155857235193, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2878365516662598, "metrics/particle/momentum/class_1/assigned/samples": 16336.0, "metrics/particle/momentum/class_1/efficiency": 0.8705579042434692, "metrics/particle/momentum/class_1/event/energy_bias": 8.50137710571289, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.896459579467773, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.30616283416748, "metrics/particle/momentum/class_1/event/px_bias": -1.9826951026916504, "metrics/particle/momentum/class_1/event/py_bias": 0.9018439054489136, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.02016830444336, "metrics/particle/momentum/class_1/event/pz_bias": -2.2089664936065674, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.518253326416016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033012937754392624, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2061692327260971, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20624637603759766, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.025746764615178108, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24554069340229034, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030179444700479507, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9454463720321655, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.021602798253297806, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.448232650756836, "metrics/particle/momentum/class_2/accepted/samples": 1226.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7555214762687683, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9632977247238159, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1341009140014648, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.711127519607544, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3402433395385742, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.4483418464660645, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06264083087444305, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0465853214263916, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.20900632441043854, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.55355167388916, "metrics/particle/momentum/class_2/assigned/samples": 2219.0, "metrics/particle/momentum/class_2/efficiency": 0.41182398796081543, "metrics/particle/momentum/class_2/event/energy_bias": -15.25594425201416, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.405881881713867, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.63381290435791, "metrics/particle/momentum/class_2/event/px_bias": 1.3018348217010498, "metrics/particle/momentum/class_2/event/py_bias": -2.00161075592041, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.402109146118164, "metrics/particle/momentum/class_2/event/pz_bias": -0.9675111770629883, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.067350387573242, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019806629046797752, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16466796398162842, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16212362051010132, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01008771825581789, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20302538573741913, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001450326293706894, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2969290614128113, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.017772536724805832, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.018972635269165, "metrics/particle/momentum/class_3/accepted/samples": 11032.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3254454731941223, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4291250705718994, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.238680124282837, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.068333864212036, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.195606991648674, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.838918685913086, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010947893373668194, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7272559404373169, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.015444244258105755, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.206866979598999, "metrics/particle/momentum/class_3/assigned/samples": 14432.0, "metrics/particle/momentum/class_3/efficiency": 0.7254554033279419, "metrics/particle/momentum/class_3/event/energy_bias": 5.298086166381836, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.859631538391113, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.580619812011719, "metrics/particle/momentum/class_3/event/px_bias": -0.14291225373744965, "metrics/particle/momentum/class_3/event/py_bias": 0.0924314558506012, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.404966354370117, "metrics/particle/momentum/class_3/event/pz_bias": 0.06989774107933044, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.213502883911133, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02510002627968788, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1735970675945282, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16321144998073578, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06804589927196503, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20423609018325806, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00749063678085804, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9582123160362244, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12777358293533325, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2579293251037598, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3067629039287567, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.720210075378418, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.337991237640381, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.225114107131958, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.18497109413146973, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.71303653717041, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017341040074825287, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.262840747833252, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.21555158495903015, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.371538162231445, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.7335164546966553, "metrics/particle/momentum/class_4/event/energy_bias": -1.104386329650879, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8955023288726807, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9764089584350586, "metrics/particle/momentum/class_4/event/px_bias": 0.021006233990192413, "metrics/particle/momentum/class_4/event/py_bias": -0.11012120544910431, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7514591217041016, "metrics/particle/momentum/class_4/event/pz_bias": 0.442141056060791, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.456077575683594, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01771879568696022, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18792253732681274, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18433625996112823, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012267479673027992, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2291727513074875, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029629629105329514, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.817784070968628, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.45082390308380127, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.6135780811309814, "metrics/particle/momentum/class_5/accepted/samples": 135.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05858482047915459, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.26184970140457153, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.252512663602829, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0760876014828682, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.10389610379934311, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3833957314491272, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06493506580591202, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5187244415283203, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3473873436450958, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.387527942657471, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.849056601524353, "metrics/particle/momentum/class_5/event/energy_bias": -1.0968308448791504, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.898996353149414, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1714859008789062, "metrics/particle/momentum/class_5/event/px_bias": -0.006865300238132477, "metrics/particle/momentum/class_5/event/py_bias": 0.2572344243526459, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5540682077407837, "metrics/particle/momentum/class_5/event/pz_bias": 0.013821035623550415, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.804804801940918, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3936411440372467, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.39501065015792847, "metrics/set_slot/angular_recall_all_dr0p10": 0.9487017393112183, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8187554478645325, "metrics/set_slot/soft_count_bias_mean": 0.3274088501930237}, "step_train_time": 5.030660629272461, "step_valid_time": 29922.551790237427, "step_total_time": 29927.582451343536, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001182845976908, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16156684441921643, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3003, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2855, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2821, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9431780640898579, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507159507159507, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9469320066334992, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04928404928404929, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319458209448299, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9393939393939394, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9356550580431178, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9431780640898579, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970611625301203, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15189010578209827, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2122, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9412024756852343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504464285714286, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9458018658374056, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049553571428571426, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9381078691423519, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9473214285714285, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9426921368280764, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9412024756852343, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9858533071454942, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13999521990799058, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1269, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1169, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122670807453416, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9259259259259259, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9190457567461869, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07407407407407407, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.907608695652174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9211977935382191, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9143527571372703, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122670807453416} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_48000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..1e823d0afa1e533182fb13bcd92cc92f0966ea38 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.274019329164298, "train": {"Classification_binary": 0.20919089019298553, "Cardinality": 0.14108456671237946, "Classification": 0.08912239223718643, "Regression_pt": 0.16891179978847504, "Regression_eta": 0.0016807718202471733, "Regression_sin_phi": 0.0008825493277981877, "Regression_cos_phi": 0.001000287476927042, "Regression_energy": 0.1681567281484604, "HitGrouping": 0.22971779108047485, "Auxiliary": 0.28761470317840576, "Total": 1.2877936363220215, "Total_unweighted": 0.6389454189920798}, "train_group": {}, "valid": {"Classification_binary": 0.182485302289327, "Cardinality": 0.12621297438939413, "Classification": 0.08798255523045857, "Regression_pt": 0.16237247983614603, "Regression_eta": 0.0016198915739854176, "Regression_sin_phi": 0.0008973710549374422, "Regression_cos_phi": 0.0009245409940679868, "Regression_energy": 0.1618956724802653, "HitGrouping": 0.2277104059855143, "Auxiliary": 0.26081828276316327, "Total": 1.2036675612131755, "Total_unweighted": 0.5981778134591877, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7488975524902344, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3716452121734619, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43449512124061584, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3635210394859314, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36222755908966064, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5717388987541199, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6877968907356262, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.323490709066391, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4079972803592682, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3149685859680176, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.349605530500412, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6638427376747131, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7639184594154358, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36417025327682495, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.406259685754776, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35761865973472595, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3467746376991272, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5452948808670044, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7522569894790649, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3629268407821655, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41550150513648987, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35569244623184204, "metrics/hit_grouping/calorimeter/all/object_purity": 0.35200121998786926, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5644376873970032, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025785135105252266, "metrics/hit_grouping/calorimeter/background_recall": 0.6319084763526917, "metrics/hit_grouping/calorimeter/completeness": 0.41756969690322876, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4414292573928833, "metrics/hit_grouping/calorimeter/energy_purity": 0.4362339377403259, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4880678355693817, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7729721069335938, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.378773033618927, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34810391068458557, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37383800745010376, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37270820140838623, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5157957673072815, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7490397691726685, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3744620382785797, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4552401602268219, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36556321382522583, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.36614561080932617, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5762979388237, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7117627263069153, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.31736505031585693, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.51674884557724, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.30699801445007324, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2934315502643585, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6550372838973999, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7379676103591919, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3927571475505829, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4999065697193146, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3849448561668396, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.38016757369041443, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6003114581108093, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7337114214897156, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4580884873867035, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6232339143753052, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.44575920701026917, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4438476264476776, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5913134217262268, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7699922919273376, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3186802566051483, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29817864298820496, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.31271907687187195, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.30996766686439514, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5224320292472839, "metrics/hit_grouping/calorimeter/purity": 0.41141781210899353, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8464701175689697, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0745936706662178, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.17076922953128815, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07241391390562057, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.035990435630083084, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05538461357355118, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7475499510765076, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14455460011959076, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12086658924818039, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1434212625026703, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13850468397140503, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.42360320687294006, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7505197525024414, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3816322684288025, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4376639425754547, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.37393927574157715, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37101614475250244, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5833879709243774, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7220855951309204, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4348475933074951, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.44715216755867004, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45515456795692444, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6147639155387878, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7719177007675171, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3123948276042938, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4792620837688446, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4999670088291168, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47167694568634033, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.672956645488739, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4445144534111023, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39694201946258545, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4010433852672577, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.683153510093689, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6968192458152771, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4303325116634369, "metrics/hit_grouping/tracker/all/object_completeness": 0.41974493861198425, "metrics/hit_grouping/tracker/all/object_purity": 0.4264775514602661, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6437275409698486, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005240089260041714, "metrics/hit_grouping/tracker/background_recall": 0.029141740873456, "metrics/hit_grouping/tracker/completeness": 0.4387987554073334, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5178737640380859, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7072179317474365, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42268791794776917, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4737642705440521, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4837511479854584, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6343507766723633, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6869991421699524, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45212259888648987, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40292423963546753, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.41217535734176636, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.66438227891922, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6841692924499512, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4241504967212677, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3177013695240021, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3145339787006378, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6425970792770386, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7190393209457397, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45533788204193115, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4415537714958191, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.43930280208587646, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6098130941390991, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7395562529563904, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4387483298778534, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4441622495651245, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4401969313621521, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5665779113769531, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6603574752807617, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39627721905708313, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3870401978492737, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40735718607902527, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7014750838279724, "metrics/hit_grouping/tracker/purity": 0.4183952212333679, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8846176862716675, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014769765548408031, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02869916707277298, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.029913511127233505, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07906168699264526, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6794219017028809, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21721099317073822, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.23124480247497559, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.230536088347435, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5795748829841614, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6732815504074097, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.511067807674408, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4943452775478363, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5026749968528748, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7277610301971436, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.43718087673187256, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5564565062522888, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6154823899269104, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5950828790664673, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.04854646325111389, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06427810341119766, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.04961395263672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30521970987319946, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17477989196777344, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8668975234031677, "metrics/particle/class_1/pid_accuracy": 0.9641430377960205, "metrics/particle/class_2/efficiency": 0.4077855050563812, "metrics/particle/class_2/pid_accuracy": 0.6063273549079895, "metrics/particle/class_3/efficiency": 0.7203355431556702, "metrics/particle/class_3/pid_accuracy": 0.9650058150291443, "metrics/particle/class_4/efficiency": 0.6977863311767578, "metrics/particle/class_4/pid_accuracy": 0.3531034588813782, "metrics/particle/class_5/efficiency": 0.8799999952316284, "metrics/particle/class_5/pid_accuracy": 0.40454545617103577, "metrics/particle/count/bias_mean": -2.720703125, "metrics/particle/count/mae": 4.284505367279053, "metrics/particle/count/prediction_mean": 67.33528900146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07349512726068497, "metrics/particle/event/energy_response_mean": 0.9858737587928772, "metrics/particle/event/met_abs_error": 8.5310640335083, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05977446958422661, "metrics/particle/event/scalar_pt_response_mean": 1.0006006956100464, "metrics/particle/event/vector_pt_closure": 0.06322797387838364, "metrics/particle/matched/delta_eta_abs_mean": 0.014212356880307198, "metrics/particle/matched/delta_phi_abs_mean": 0.01443817000836134, "metrics/particle/matched/delta_r_mean": 0.02268548123538494, "metrics/particle/matched/energy_relative_abs_mean": 0.1599380373954773, "metrics/particle/matched/pid_accuracy": 0.9402847290039062, "metrics/particle/matched/pt_relative_abs_mean": 0.15916268527507782, "metrics/particle/matching/duplicate_fraction": 0.06625929474830627, "metrics/particle/matching/efficiency": 0.7638049721717834, "metrics/particle/matching/f1": 0.7789303064346313, "metrics/particle/matching/purity": 0.7946667671203613, "metrics/particle/matching/target_coverage_dr0p05": 0.7272363901138306, "metrics/particle/matching/target_coverage_dr0p10": 0.8250376582145691, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5378510355949402, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8015445470809937, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02208038792014122, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1576659083366394, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15700143575668335, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012599043548107147, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19780874252319336, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010589094832539558, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8029334545135498, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02354375459253788, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.169281005859375, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8216.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.275056391954422, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.15943443775177, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3982493877410889, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2258294820785522, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15241087973117828, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.6384220123291, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020426200702786446, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2127424478530884, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.035485755652189255, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3110296726226807, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10183.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7512801885604858, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3337018489837646, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.72646713256836, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.409834861755371, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5570724606513977, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.49143317341804504, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.357833862304688, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.013325557112693787, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.014103889465332, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022792229428887367, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16094686090946198, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16065041720867157, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023398345336318016, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20236928761005402, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.033224400132894516, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.68901789188385, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.12279265373945236, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.049281120300293, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1836.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.47599825263023376, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5367900133132935, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.3031463623046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.1328864097595215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20794861018657684, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 28.51779556274414, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06583701074123383, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.850463390350342, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08080834895372391, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.9438066482543945, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2491.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6077457666397095, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.582498073577881, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.717117309570312, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2584209442138672, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.13226664066314697, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11580222845077515, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.54941177368164, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.11057823151350021, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.0659122467041, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022918863222002983, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16022972762584686, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1595231145620346, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016855232417583466, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1995539516210556, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006477446877397597, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27358707785606384, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007895995862782001, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8023492693901062, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16982.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22711355984210968, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3541754484176636, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1023246049880981, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9276445508003235, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15059520304203033, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.08041763305664, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005163264460861683, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4866299629211426, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0067082676105201244, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8934705257415771, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20917.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7921817302703857, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.121450662612915, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.758331298828125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.118349075317383, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.859660267829895, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.9438343048095703, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.724848747253418, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9939961433410645, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.698820114135742, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022655436769127846, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15949925780296326, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15883329510688782, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016006089746952057, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19921813905239105, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005881482735276222, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5305908918380737, "metrics/particle/momentum/all/accepted/pz_bias": -0.003775909310206771, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.717008113861084, "metrics/particle/momentum/all/accepted/samples": 27034.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2601037323474884, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3086823225021362, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.355238914489746, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1815718412399292, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15539877116680145, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.855412483215332, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014289542101323605, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8820427060127258, "metrics/particle/momentum/all/assigned/pz_bias": 0.0005876875366084278, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1992931365966797, "metrics/particle/momentum/all/assigned/samples": 33591.0, "metrics/particle/momentum/all/efficiency": 0.7638017535209656, "metrics/particle/momentum/all/event/energy_bias": -3.7947494983673096, "metrics/particle/momentum/all/event/p3_error_mean": 21.584163665771484, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.363871574401855, "metrics/particle/momentum/all/event/px_bias": 2.548999071121216, "metrics/particle/momentum/all/event/py_bias": 2.551069974899292, "metrics/particle/momentum/all/event/pz_abs_mean": 15.514969825744629, "metrics/particle/momentum/all/event/pz_bias": -1.117899775505066, "metrics/particle/momentum/all/event/pz_residual_rms": 24.94542694091797, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02424376830458641, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15225203335285187, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15211747586727142, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02419130690395832, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19139595329761505, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006277160719037056, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5693524479866028, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.011248071677982807, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.743149995803833, "metrics/particle/momentum/class_1/accepted/samples": 14497.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12164022773504257, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.34028637409210205, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.36408358812332153, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21816574037075043, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08718262612819672, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.5728065967559814, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010278373025357723, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7251100540161133, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005025823134928942, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.339510917663574, "metrics/particle/momentum/class_1/assigned/samples": 16345.0, "metrics/particle/momentum/class_1/efficiency": 0.8687601089477539, "metrics/particle/momentum/class_1/event/energy_bias": 7.65001106262207, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.739654541015625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.443893432617188, "metrics/particle/momentum/class_1/event/px_bias": 0.9302768707275391, "metrics/particle/momentum/class_1/event/py_bias": 1.341408133506775, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.825264930725098, "metrics/particle/momentum/class_1/event/pz_bias": -1.6612125635147095, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.881093978881836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03267231583595276, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2047961950302124, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2051784247159958, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0276816263794899, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2452750951051712, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03269069641828537, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8722405433654785, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.14813938736915588, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.211035251617432, "metrics/particle/momentum/class_2/accepted/samples": 1193.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7684738039970398, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9104301929473877, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1364662647247314, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7264525890350342, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.349709689617157, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.1558051109313965, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06297454237937927, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9422435760498047, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1073521077632904, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.972256660461426, "metrics/particle/momentum/class_2/assigned/samples": 2239.0, "metrics/particle/momentum/class_2/efficiency": 0.40073898434638977, "metrics/particle/momentum/class_2/event/energy_bias": -14.23831558227539, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.654996871948242, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.821985244750977, "metrics/particle/momentum/class_2/event/px_bias": 1.1509121656417847, "metrics/particle/momentum/class_2/event/py_bias": 0.6280949115753174, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.41673755645752, "metrics/particle/momentum/class_2/event/pz_bias": 1.2776257991790771, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.676807403564453, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01950405165553093, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16354572772979736, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16230490803718567, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.009273789823055267, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20331349968910217, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002194185508415103, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.30366572737693787, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.011675011366605759, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0248873233795166, "metrics/particle/momentum/class_3/accepted/samples": 10938.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3387988209724426, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.390066385269165, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4544200897216797, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.284775972366333, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20182093977928162, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.0054874420166, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010967030189931393, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.717597484588623, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.023432886227965355, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.099396228790283, "metrics/particle/momentum/class_3/assigned/samples": 14498.0, "metrics/particle/momentum/class_3/efficiency": 0.7192740440368652, "metrics/particle/momentum/class_3/event/energy_bias": 5.440351963043213, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.499429702758789, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.086447715759277, "metrics/particle/momentum/class_3/event/px_bias": 0.3504346013069153, "metrics/particle/momentum/class_3/event/py_bias": 0.2592759430408478, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.745572090148926, "metrics/particle/momentum/class_3/event/pz_bias": -1.0861852169036865, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.815542221069336, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024196460843086243, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1727767139673233, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16316817700862885, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05307507887482643, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20606820285320282, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00749063678085804, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.007745623588562, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.24131423234939575, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.310274839401245, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3121393024921417, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.730235576629639, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.976797342300415, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.8629679679870605, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19658119976520538, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.29485511779785, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011396011337637901, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1347601413726807, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1848781406879425, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.613614320755005, "metrics/particle/momentum/class_4/assigned/samples": 351.0, "metrics/particle/momentum/class_4/efficiency": 0.7335164546966553, "metrics/particle/momentum/class_4/event/energy_bias": -1.3769402503967285, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8713409900665283, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9745789766311646, "metrics/particle/momentum/class_4/event/px_bias": 0.06790317595005035, "metrics/particle/momentum/class_4/event/py_bias": -0.0195561945438385, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7178349494934082, "metrics/particle/momentum/class_4/event/pz_bias": 0.46694234013557434, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.253570556640625, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016052987426519394, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18265292048454285, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17998215556144714, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.00414587976410985, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2181652933359146, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02158273383975029, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9132782220840454, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.37751439213752747, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3711490631103516, "metrics/particle/momentum/class_5/accepted/samples": 139.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04339708387851715, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.30393528938293457, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.30569037795066833, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1085360050201416, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.10759493708610535, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9709293842315674, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.050632912665605545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.449756145477295, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.17676995694637299, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.933951377868652, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.8742138147354126, "metrics/particle/momentum/class_5/event/energy_bias": -1.2698562145233154, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7120251655578613, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.039595365524292, "metrics/particle/momentum/class_5/event/px_bias": 0.04947235435247421, "metrics/particle/momentum/class_5/event/py_bias": 0.3418470025062561, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4352500438690186, "metrics/particle/momentum/class_5/event/pz_bias": -0.11507029831409454, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.484052658081055, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39542126655578613, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.39696717262268066, "metrics/set_slot/angular_recall_all_dr0p10": 0.9474843144416809, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8155028820037842, "metrics/set_slot/soft_count_bias_mean": 0.32908129692077637}, "step_train_time": 4.977629899978638, "step_valid_time": 32526.069830417633, "step_total_time": 32531.04746079445, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0106969397990682, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17087777929763903, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3007, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2853, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2817, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9425173439048563, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487861656135683, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456413655949619, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05121383438643166, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9306243805748265, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9368141004323246, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337089824328804, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9425173439048563, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0059608620550324, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15683686454569767, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2118, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2110, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363395225464191, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9451137884872824, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9407061958694204, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05488621151271754, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9328028293545535, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9415439535921464, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9371530091050411, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363395225464191, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9961208719506092, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14854786221473737, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9184782608695652, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9256651017214398, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9220576773187841, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07433489827856025, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9107142857142857, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9178403755868545, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9142634450506625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9184782608695652} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_49000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..1df5f3964863e7ec48de5633a30f4d98098f5c4e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.3213947318552206, "train": {"Classification_binary": 0.21643780171871185, "Cardinality": 0.1384437531232834, "Classification": 0.09342502057552338, "Regression_pt": 0.16524191200733185, "Regression_eta": 0.0017133232904598117, "Regression_sin_phi": 0.0009352728957310319, "Regression_cos_phi": 0.0009044528705999255, "Regression_energy": 0.16415463387966156, "HitGrouping": 0.23090343177318573, "Auxiliary": 0.28671255707740784, "Total": 1.2892229557037354, "Total_unweighted": 0.6428124172380194}, "train_group": {}, "valid": {"Classification_binary": 0.1794584592183431, "Cardinality": 0.12756248315175375, "Classification": 0.08763155341148376, "Regression_pt": 0.1637901266415914, "Regression_eta": 0.0015938348757723968, "Regression_sin_phi": 0.0008867386107643446, "Regression_cos_phi": 0.0009090315240124861, "Regression_energy": 0.163127730290095, "HitGrouping": 0.2281109094619751, "Auxiliary": 0.25985856850941974, "Total": 1.1996346314748128, "Total_unweighted": 0.5973974745720625, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7468632459640503, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3696712553501129, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42212146520614624, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.36072659492492676, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.36456480622291565, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.578067421913147, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.690685510635376, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3239385783672333, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41172483563423157, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.31613120436668396, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.356624960899353, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6597763299942017, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7644906640052795, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36526042222976685, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40254107117652893, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35852697491645813, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3528536856174469, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.546637773513794, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7521982192993164, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3629733920097351, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.40966564416885376, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35543206334114075, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3569605350494385, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5668997168540955, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025894083082675934, "metrics/hit_grouping/calorimeter/background_recall": 0.6333335638046265, "metrics/hit_grouping/calorimeter/completeness": 0.41392982006073, "metrics/hit_grouping/calorimeter/energy_completeness": 0.43501371145248413, "metrics/hit_grouping/calorimeter/energy_purity": 0.42992812395095825, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4792613089084625, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7713878154754639, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.38075122237205505, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3424902856349945, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3754832446575165, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37987056374549866, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.519199788570404, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7477098703384399, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.37742477655410767, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4492964446544647, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3678617775440216, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3719545900821686, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5792091488838196, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7162575721740723, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.310283899307251, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5105549097061157, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3000374138355255, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29284220933914185, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6550372838973999, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7404935956001282, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3894374966621399, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4941139817237854, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3817893862724304, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.38116317987442017, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5951417088508606, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7477658987045288, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.45594340562820435, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6263736486434937, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.445600301027298, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4395581781864166, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5729984045028687, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7653486132621765, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32262614369392395, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29114770889282227, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3155581057071686, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3203716576099396, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.535757303237915, "metrics/hit_grouping/calorimeter/purity": 0.4078771770000458, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8368993401527405, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07378636300563812, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1569230705499649, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07234065979719162, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03658619150519371, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05846153944730759, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.75355464220047, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1541527658700943, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12371721863746643, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.15242458879947662, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14968092739582062, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.41562142968177795, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7503147721290588, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.38114771246910095, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4314992129802704, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3731421232223511, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37571093440055847, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5864375829696655, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7190371751785278, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4429169297218323, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.442867249250412, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4563573896884918, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.621637761592865, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7775708436965942, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3230510354042053, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.480935662984848, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.49088937044143677, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47952887415885925, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6720954179763794, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4395526647567749, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3953660726547241, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4039210081100464, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6866976618766785, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6958153247833252, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4308048486709595, "metrics/hit_grouping/tracker/all/object_completeness": 0.41758638620376587, "metrics/hit_grouping/tracker/all/object_purity": 0.4278189539909363, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.648686945438385, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005618303082883358, "metrics/hit_grouping/tracker/background_recall": 0.030385594815015793, "metrics/hit_grouping/tracker/completeness": 0.4369979500770569, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5168034434318542, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7064109444618225, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4244191348552704, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47251802682876587, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4864386022090912, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6432802081108093, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6862920522689819, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4532648026943207, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4047706425189972, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.41904938220977783, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6592423319816589, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.68241286277771, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42111650109291077, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3092624843120575, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3070092797279358, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6494740843772888, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7172076106071472, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4499460756778717, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4378294348716736, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.440489262342453, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6164630055427551, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7413482666015625, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4400798976421356, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4400446116924286, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43423137068748474, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.565912127494812, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6599403023719788, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40423789620399475, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3872659206390381, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41018542647361755, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7052212357521057, "metrics/hit_grouping/tracker/purity": 0.4168366491794586, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8816488981246948, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013900956138968468, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02687973715364933, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.031373172998428345, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08514335006475449, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6732422709465027, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2187661975622177, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2198100984096527, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22378242015838623, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5852773189544678, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.673133909702301, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.511599063873291, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4931968152523041, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5049222111701965, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7324833273887634, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3788255453109741, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5614389181137085, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6218132972717285, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6000999212265015, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.10690178722143173, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06929515302181244, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.185054779052734, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2955699563026428, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16513016819953918, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8637910485267639, "metrics/particle/class_1/pid_accuracy": 0.9661902189254761, "metrics/particle/class_2/efficiency": 0.415974885225296, "metrics/particle/class_2/pid_accuracy": 0.6299892067909241, "metrics/particle/class_3/efficiency": 0.7142213582992554, "metrics/particle/class_3/pid_accuracy": 0.9632644653320312, "metrics/particle/class_4/efficiency": 0.713185727596283, "metrics/particle/class_4/pid_accuracy": 0.37516868114471436, "metrics/particle/class_5/efficiency": 0.8700000047683716, "metrics/particle/class_5/pid_accuracy": 0.4206896424293518, "metrics/particle/count/bias_mean": -3.5924479961395264, "metrics/particle/count/mae": 4.611979007720947, "metrics/particle/count/prediction_mean": 66.46353912353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.0728711411356926, "metrics/particle/event/energy_response_mean": 0.9724165797233582, "metrics/particle/event/met_abs_error": 7.7348952293396, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05952347442507744, "metrics/particle/event/scalar_pt_response_mean": 0.9847103953361511, "metrics/particle/event/vector_pt_closure": 0.062045320868492126, "metrics/particle/matched/delta_eta_abs_mean": 0.014306043274700642, "metrics/particle/matched/delta_phi_abs_mean": 0.014230705797672272, "metrics/particle/matched/delta_r_mean": 0.02262663096189499, "metrics/particle/matched/energy_relative_abs_mean": 0.16120877861976624, "metrics/particle/matched/pid_accuracy": 0.9415143132209778, "metrics/particle/matched/pt_relative_abs_mean": 0.1602153778076172, "metrics/particle/matching/duplicate_fraction": 0.0582340732216835, "metrics/particle/matching/efficiency": 0.7604780197143555, "metrics/particle/matching/f1": 0.7804896831512451, "metrics/particle/matching/purity": 0.801582932472229, "metrics/particle/matching/target_coverage_dr0p05": 0.724615752696991, "metrics/particle/matching/target_coverage_dr0p10": 0.82228684425354, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5324238538742065, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7973626255989075, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021947219967842102, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15661495923995972, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1560082882642746, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014789309352636337, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1962329000234604, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008584743365645409, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8123019933700562, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.015164967626333237, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1867871284484863, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8154.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2595207989215851, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2868461608886719, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.240300178527832, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.067501425743103, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1556921899318695, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.56205940246582, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019139230251312256, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.197584867477417, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014669770374894142, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2610104084014893, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10084.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7456108331680298, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.8561400175094604, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.113521575927734, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.2237629890441895, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15819908678531647, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4939495027065277, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.705592632293701, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.8977206349372864, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.806304931640625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02283170260488987, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1621559113264084, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16332989931106567, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023759355768561363, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20431064069271088, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0312331710010767, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6807087659835815, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.183670774102211, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.360383033752441, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1857.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4541971981525421, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.093717336654663, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.6893857717514038, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.5204569101333618, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1942416876554489, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.350905418395996, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05961070582270622, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7146763801574707, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19779321551322937, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.559990882873535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2466.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6146970987319946, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.396154880523682, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.707137107849121, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3163342475891113, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08131716400384903, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0035338602028787136, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.55250358581543, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.711932897567749, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.777502059936523, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022654760628938675, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16197308897972107, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16044798493385315, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01853199116885662, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20087061822414398, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006515815621241927, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26976537704467773, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0001935458421939984, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7772684693336487, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16882.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2147429883480072, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2258265018463135, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9661380052566528, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7890855073928833, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14666280150413513, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.660826683044434, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004488200414925814, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4804671108722687, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.020924683660268784, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9594093561172485, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20721.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7875168919563293, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.499131441116333, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.961870193481445, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.363744735717773, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.656362533569336, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.45018312335014343, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.498350620269775, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.455636203289032, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.61429500579834, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022452451288700104, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1603611409664154, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15930086374282837, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017758160829544067, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1997169405221939, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005168631207197905, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5316909551620483, "metrics/particle/momentum/all/accepted/pz_bias": -0.017159271985292435, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7725809812545776, "metrics/particle/momentum/all/accepted/samples": 26893.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24606257677078247, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.23452889919281, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1028391122817993, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9276780486106873, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15292596817016602, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.766688346862793, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013014337047934532, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8634124398231506, "metrics/particle/momentum/all/assigned/pz_bias": -0.02324572205543518, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.138519287109375, "metrics/particle/momentum/all/assigned/samples": 33271.0, "metrics/particle/momentum/all/efficiency": 0.7598180770874023, "metrics/particle/momentum/all/event/energy_bias": -7.751426696777344, "metrics/particle/momentum/all/event/p3_error_mean": 20.698944091796875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.620953559875488, "metrics/particle/momentum/all/event/px_bias": 1.8958789110183716, "metrics/particle/momentum/all/event/py_bias": -0.9405988454818726, "metrics/particle/momentum/all/event/pz_abs_mean": 15.264825820922852, "metrics/particle/momentum/all/event/pz_bias": -3.0652899742126465, "metrics/particle/momentum/all/event/pz_residual_rms": 24.338075637817383, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023337023332715034, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15371426939964294, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1532585620880127, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.019933216273784637, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19277825951576233, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005693258251994848, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5786681771278381, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03547941520810127, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7751092910766602, "metrics/particle/momentum/class_1/accepted/samples": 14403.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11109082400798798, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3376919627189636, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.38011765480041504, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23184627294540405, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0885227769613266, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.7125356197357178, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008483433164656162, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7159961462020874, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01622517965734005, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.239595413208008, "metrics/particle/momentum/class_1/assigned/samples": 16267.0, "metrics/particle/momentum/class_1/efficiency": 0.8631269931793213, "metrics/particle/momentum/class_1/event/energy_bias": 5.049848556518555, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.95956039428711, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.62480640411377, "metrics/particle/momentum/class_1/event/px_bias": -0.06417419016361237, "metrics/particle/momentum/class_1/event/py_bias": 1.0555565357208252, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.006078720092773, "metrics/particle/momentum/class_1/event/pz_bias": -1.7234132289886475, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.22344398498535, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03244469687342644, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19599846005439758, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19702807068824768, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.024593347683548927, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23627302050590515, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028949545696377754, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7783942222595215, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.02604106068611145, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.3657073974609375, "metrics/particle/momentum/class_2/accepted/samples": 1209.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6765931248664856, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8250820636749268, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0457277297973633, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6563570499420166, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.32774674892425537, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.875663757324219, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05540037155151367, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.619785785675049, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.14378362894058228, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.997658729553223, "metrics/particle/momentum/class_2/assigned/samples": 2148.0, "metrics/particle/momentum/class_2/efficiency": 0.40611353516578674, "metrics/particle/momentum/class_2/event/energy_bias": -13.521729469299316, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.55935287475586, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.818214416503906, "metrics/particle/momentum/class_2/event/px_bias": 1.7538237571716309, "metrics/particle/momentum/class_2/event/py_bias": -2.198535442352295, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.488428115844727, "metrics/particle/momentum/class_2/event/pz_bias": -0.8440090417861938, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.818838119506836, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02020557038486004, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16459998488426208, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16272787749767303, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01891409605741501, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20375961065292358, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018385732546448708, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.30571433901786804, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0022585156839340925, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1182122230529785, "metrics/particle/momentum/class_3/accepted/samples": 10878.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.336312860250473, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.2741246223449707, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.896727204322815, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.722186803817749, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19934503734111786, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.963196754455566, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011566332541406155, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7454383373260498, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.014270092360675335, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.368201971054077, "metrics/particle/momentum/class_3/assigned/samples": 14352.0, "metrics/particle/momentum/class_3/efficiency": 0.7153284549713135, "metrics/particle/momentum/class_3/event/energy_bias": 3.337521553039551, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.648750305175781, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.426252841949463, "metrics/particle/momentum/class_3/event/px_bias": 0.22679124772548676, "metrics/particle/momentum/class_3/event/py_bias": -0.03233423829078674, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.286150932312012, "metrics/particle/momentum/class_3/event/pz_bias": -0.7634724974632263, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.973764419555664, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024287015199661255, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17223119735717773, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16141200065612793, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05598387494683266, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20611728727817535, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8333519101142883, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2604324519634247, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7506145238876343, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2409505844116211, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.244657039642334, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.8710126876831055, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.755657911300659, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17630058526992798, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.31943702697754, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005780346691608429, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9771239757537842, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10847672820091248, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.310936212539673, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.7335164546966553, "metrics/particle/momentum/class_4/event/energy_bias": -1.2338980436325073, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.9122769832611084, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0379459857940674, "metrics/particle/momentum/class_4/event/px_bias": 0.06097676604986191, "metrics/particle/momentum/class_4/event/py_bias": -0.00017854198813438416, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7091253995895386, "metrics/particle/momentum/class_4/event/pz_bias": 0.2750492990016937, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.453681945800781, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01606035605072975, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18513813614845276, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18556898832321167, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0036012863274663687, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22595854103565216, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.014705882407724857, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9563544988632202, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09613466262817383, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.485278606414795, "metrics/particle/momentum/class_5/accepted/samples": 136.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10240849107503891, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3013356626033783, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.30231043696403503, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08349601924419403, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.13924050331115723, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5599762797355652, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.050632912665605545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.630190134048462, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.21110650897026062, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.15690803527832, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.8553459048271179, "metrics/particle/momentum/class_5/event/energy_bias": -1.383169174194336, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.696138858795166, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0203280448913574, "metrics/particle/momentum/class_5/event/px_bias": -0.08153889328241348, "metrics/particle/momentum/class_5/event/py_bias": 0.23489263653755188, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4119884967803955, "metrics/particle/momentum/class_5/event/pz_bias": -0.009444288909435272, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.424493789672852, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.394385427236557, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3956149220466614, "metrics/set_slot/angular_recall_all_dr0p10": 0.9490177035331726, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8144434094429016, "metrics/set_slot/soft_count_bias_mean": -0.32492583990097046}, "step_train_time": 6.266035795211792, "step_valid_time": 35137.16474342346, "step_total_time": 35143.43077969551, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954118032253925, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15971433830487414, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3014, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2854, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2824, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9428477039973571, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.946914399469144, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9448766760470121, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05308560053085601, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9329369012223323, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9369608493696084, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9349445456050323, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9428477039973571, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9963204102041879, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15984750151673288, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2121, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9412024756852343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9483296213808463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9447526070556912, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051670378619153676, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9376657824933687, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447661469933185, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9412025737741291, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9412024756852343, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960636954008786, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14809176164975502, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1172, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9153726708074534, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9239811912225705, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9196567862714509, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07601880877742946, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9099378881987578, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9184952978056427, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9141965678627145, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9153726708074534} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_5000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..a245db867de6d620ec70738461a8f02f739dbf2a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23687701345461437, "train": {"Classification_binary": 0.35183846950531006, "Cardinality": 0.28009843826293945, "Classification": 0.2560775876045227, "Regression_pt": 0.41880002617836, "Regression_eta": 0.0049495818093419075, "Regression_sin_phi": 0.0024409228935837746, "Regression_cos_phi": 0.0026731640100479126, "Regression_energy": 0.41411781311035156, "HitGrouping": 0.28864407539367676, "Auxiliary": 0.4927078187465668, "Total": 2.535339832305908, "Total_unweighted": 1.450897565111518}, "train_group": {}, "valid": {"Classification_binary": 0.3436930179595947, "Cardinality": 0.2604156931241353, "Classification": 0.25466181834538776, "Regression_pt": 0.4204241832097371, "Regression_eta": 0.005200169049203396, "Regression_sin_phi": 0.0026271060730020204, "Regression_cos_phi": 0.0026343483477830887, "Regression_energy": 0.4156715472539266, "HitGrouping": 0.2910100221633911, "Auxiliary": 0.47884953022003174, "Total": 2.5165379842122397, "Total_unweighted": 1.4449121902386346, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7671157121658325, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.1308155655860901, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3215264081954956, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.12708449363708496, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.09897740185260773, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5720223188400269, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7832083106040955, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.1142958551645279, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.23280243575572968, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.10977045446634293, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.07624270766973495, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5079634189605713, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7468626499176025, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.12881942093372345, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.30296456813812256, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.12473079562187195, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.1098385825753212, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6238508224487305, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7566398978233337, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.1281590759754181, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.30264437198638916, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.12414631992578506, "metrics/hit_grouping/calorimeter/all/object_purity": 0.1033301055431366, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5967780947685242, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025486022233963013, "metrics/hit_grouping/calorimeter/background_recall": 0.6247586607933044, "metrics/hit_grouping/calorimeter/completeness": 0.22394709289073944, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2654532194137573, "metrics/hit_grouping/calorimeter/energy_purity": 0.26239070296287537, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.644449770450592, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7754146456718445, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.12479427456855774, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2482532113790512, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.1215440109372139, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.11296940594911575, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5524036884307861, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7486878037452698, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.1326621025800705, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.330058217048645, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.12826202809810638, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.10112602263689041, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6162057518959045, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.725187361240387, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.13058863580226898, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3892049193382263, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.12536554038524628, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.0852237194776535, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6704588532447815, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7285982370376587, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.1537628173828125, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.39676111936569214, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.14814847707748413, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.1198500320315361, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6609779000282288, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7509907484054565, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3239800035953522, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6640502214431763, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.32011231780052185, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.24870337545871735, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6085819005966187, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7875096797943115, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.0755755677819252, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1552162915468216, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.07326607406139374, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.06696774065494537, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5262488126754761, "metrics/hit_grouping/calorimeter/purity": 0.22058004140853882, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8931306600570679, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0442764088511467, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14000000059604645, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04287179559469223, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0053175464272499084, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8103039860725403, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.035232242196798325, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.07867731153964996, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.034663598984479904, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.020601239055395126, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38768529891967773, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7506442070007324, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.13529172539710999, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.31899264454841614, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.1310252845287323, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.11017738282680511, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6205075979232788, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6952364444732666, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4438135027885437, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.30686652660369873, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2540070414543152, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6734907627105713, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7793377041816711, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.37521031498908997, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.33825650811195374, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.24025751650333405, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4784071743488312, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6355096101760864, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42466723918914795, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2650829255580902, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.23309646546840668, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.736000657081604, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6665003895759583, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4265539348125458, "metrics/hit_grouping/tracker/all/object_completeness": 0.28445208072662354, "metrics/hit_grouping/tracker/all/object_purity": 0.24030889570713043, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6945493817329407, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.003504576627165079, "metrics/hit_grouping/tracker/background_recall": 0.01451163925230503, "metrics/hit_grouping/tracker/completeness": 0.2994462549686432, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6644048690795898, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6596649885177612, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4493849575519562, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.34183308482170105, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3062129020690918, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7120729088783264, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6658991575241089, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4241385757923126, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2569456994533539, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.20972943305969238, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6946506500244141, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.682313084602356, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3784385025501251, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.18629971146583557, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.12650178372859955, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6555420756340027, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6979784965515137, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45201292634010315, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.29679998755455017, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.23555901646614075, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6593278050422668, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7217676043510437, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.42010653018951416, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.26605889201164246, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2143605500459671, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.595872163772583, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6157748103141785, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39452117681503296, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.27160021662712097, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2510593831539154, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7577850818634033, "metrics/hit_grouping/tracker/purity": 0.28502514958381653, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8960672616958618, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012163336388766766, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.013424144126474857, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.010442782193422318, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07080799341201782, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6833768486976624, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.14152410626411438, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.08703998476266861, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.06704199314117432, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5780196785926819, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6333845257759094, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5153178572654724, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3437587320804596, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.2912730574607849, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7925742268562317, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.17991787195205688, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.569805383682251, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6281939744949341, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5421119332313538, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.30580949783325195, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.011307095177471638, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.385202407836914, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.18756195902824402, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8062499761581421, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.05712215602397919, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5059754848480225, "metrics/particle/class_1/pid_accuracy": 0.8843266367912292, "metrics/particle/class_2/efficiency": 0.3037917912006378, "metrics/particle/class_2/pid_accuracy": 0.11927621811628342, "metrics/particle/class_3/efficiency": 0.4217063784599304, "metrics/particle/class_3/pid_accuracy": 0.8866052627563477, "metrics/particle/class_4/efficiency": 0.46005773544311523, "metrics/particle/class_4/pid_accuracy": 0.2447698712348938, "metrics/particle/class_5/efficiency": 0.4860000014305115, "metrics/particle/class_5/pid_accuracy": 0.1563785970211029, "metrics/particle/count/bias_mean": -12.34375, "metrics/particle/count/mae": 12.645833015441895, "metrics/particle/count/prediction_mean": 57.71223831176758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.12533937394618988, "metrics/particle/event/energy_response_mean": 0.9856805801391602, "metrics/particle/event/met_abs_error": 13.937361717224121, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10995181649923325, "metrics/particle/event/scalar_pt_response_mean": 1.0177229642868042, "metrics/particle/event/vector_pt_closure": 0.10700878500938416, "metrics/particle/matched/delta_eta_abs_mean": 0.02427317388355732, "metrics/particle/matched/delta_phi_abs_mean": 0.021776249632239342, "metrics/particle/matched/delta_r_mean": 0.03634766861796379, "metrics/particle/matched/energy_relative_abs_mean": 0.20284897089004517, "metrics/particle/matched/pid_accuracy": 0.8327510356903076, "metrics/particle/matched/pt_relative_abs_mean": 0.19938752055168152, "metrics/particle/matching/duplicate_fraction": 0.10566748678684235, "metrics/particle/matching/efficiency": 0.4521866738796234, "metrics/particle/matching/f1": 0.4958726465702057, "metrics/particle/matching/purity": 0.5489023923873901, "metrics/particle/matching/target_coverage_dr0p05": 0.3778042197227478, "metrics/particle/matching/target_coverage_dr0p10": 0.5471349358558655, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.27985429763793945, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5228611826896667, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0355384536087513, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1977747231721878, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19680523872375488, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.016685321927070618, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23833666741847992, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.020132791250944138, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.4362051486968994, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07619880139827728, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.2682371139526367, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4669.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6158350110054016, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.6176440715789795, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.045351028442383, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.845132350921631, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.35032472014427185, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.85108757019043, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04043621942400932, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.25567626953125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06614438444375992, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.050137996673584, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8161.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.42693856358528137, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2317694127559662, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.683521270751953, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.81375789642334, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.1813191175460815, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.0010311603546143, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.035834312438965, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.633701801300049, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 22.79267692565918, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03352131322026253, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1890362948179245, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18875649571418762, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0045319776982069016, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22957639396190643, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.05324298143386841, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.5466933250427246, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0011797643965110183, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.203036785125732, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1033.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.744938850402832, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8647199869155884, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.292942523956299, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.112850189208984, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.35845986008644104, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.945096969604492, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09436008334159851, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.7187108993530273, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.20517517626285553, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.634345054626465, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1844.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3419397473335266, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.95059871673584, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.804839134216309, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.3970699310302734, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04772936552762985, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.015117567032575607, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 14.498712539672852, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.5145630836486816, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 32.69524383544922, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03703005984425545, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20491065084934235, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19967344403266907, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.017806820571422577, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23963378369808197, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0025191358290612698, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4912256598472595, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.026449210941791534, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2510550022125244, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 10321.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6194722652435303, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.058765411376953, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.2908737659454346, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.093946695327759, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.37058883905410767, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.136545181274414, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.017749007791280746, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.2088958024978638, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.004636277910321951, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.6151256561279297, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19156.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.48145729303359985, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 8.904621124267578, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.49257469177246, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.529658317565918, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.6114065647125244, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.654211044311523, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.003538131713867, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.43061158061027527, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.93643569946289, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0363692045211792, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20180784165859222, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19813384115695953, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.016624195501208305, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23861989378929138, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.010921799577772617, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8991022706031799, "metrics/particle/momentum/all/accepted/pz_bias": -0.005090885795652866, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.4219720363616943, "metrics/particle/momentum/all/accepted/samples": 16023.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6263882517814636, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.5167126655578613, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.285527467727661, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.0887441635131836, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3641507625579834, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.052337646484375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.028942765668034554, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.6605563163757324, "metrics/particle/momentum/all/assigned/pz_bias": 0.03453104570508003, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.4054789543151855, "metrics/particle/momentum/all/assigned/samples": 29161.0, "metrics/particle/momentum/all/efficiency": 0.4527038335800171, "metrics/particle/momentum/all/event/energy_bias": 1.1857922077178955, "metrics/particle/momentum/all/event/p3_error_mean": 34.75231170654297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.90345573425293, "metrics/particle/momentum/all/event/px_bias": 3.7449960708618164, "metrics/particle/momentum/all/event/py_bias": -5.6703596115112305, "metrics/particle/momentum/all/event/pz_abs_mean": 25.389209747314453, "metrics/particle/momentum/all/event/pz_bias": -0.6885270476341248, "metrics/particle/momentum/all/event/pz_residual_rms": 38.07870864868164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04513111710548401, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21774281561374664, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21358224749565125, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.01787392608821392, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2532714009284973, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.012533995322883129, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.0342586040496826, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.013972374610602856, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.5064876079559326, "metrics/particle/momentum/class_1/accepted/samples": 8457.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.4479345381259918, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.8169533014297485, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.946730136871338, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.7424142360687256, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.32883909344673157, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.326329231262207, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.023992255330085754, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.543419361114502, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04741302877664566, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.779170513153076, "metrics/particle/momentum/class_1/assigned/samples": 14463.0, "metrics/particle/momentum/class_1/efficiency": 0.5068017244338989, "metrics/particle/momentum/class_1/event/energy_bias": 31.010427474975586, "metrics/particle/momentum/class_1/event/p3_error_mean": 30.931734085083008, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.47684097290039, "metrics/particle/momentum/class_1/event/px_bias": 2.1223385334014893, "metrics/particle/momentum/class_1/event/py_bias": -2.794497013092041, "metrics/particle/momentum/class_1/event/pz_abs_mean": 21.778047561645508, "metrics/particle/momentum/class_1/event/pz_bias": 2.203869581222534, "metrics/particle/momentum/class_1/event/pz_residual_rms": 31.17466926574707, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04680138826370239, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2290484458208084, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22529812157154083, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05978868156671524, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2652500867843628, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.042222220450639725, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.164546489715576, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04807484149932861, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.064192295074463, "metrics/particle/momentum/class_2/accepted/samples": 900.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7191744446754456, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8060243129730225, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9177967309951782, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5195857882499695, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3395165801048279, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.908625841140747, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06295672059059143, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.012815475463867, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07847613096237183, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.837071895599365, "metrics/particle/momentum/class_2/assigned/samples": 1779.0, "metrics/particle/momentum/class_2/efficiency": 0.3023177683353424, "metrics/particle/momentum/class_2/event/energy_bias": -30.19184684753418, "metrics/particle/momentum/class_2/event/p3_error_mean": 30.21110725402832, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.815242767333984, "metrics/particle/momentum/class_2/event/px_bias": 1.23052179813385, "metrics/particle/momentum/class_2/event/py_bias": -0.8337119817733765, "metrics/particle/momentum/class_2/event/pz_abs_mean": 22.56203269958496, "metrics/particle/momentum/class_2/event/pz_bias": -0.8671417832374573, "metrics/particle/momentum/class_2/event/pz_residual_rms": 34.22874069213867, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0230896957218647, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17634671926498413, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17330296337604523, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007767952047288418, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21291516721248627, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003588143503293395, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.497650146484375, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.031218629330396652, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4828860759735107, "metrics/particle/momentum/class_3/accepted/samples": 6410.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8234891295433044, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.883861064910889, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.203145980834961, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.045650005340576, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.40910911560058594, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.615345001220703, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02913631685078144, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5773463249206543, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02385411597788334, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.60589599609375, "metrics/particle/momentum/class_3/assigned/samples": 12493.0, "metrics/particle/momentum/class_3/efficiency": 0.42151641845703125, "metrics/particle/momentum/class_3/event/energy_bias": 6.139992713928223, "metrics/particle/momentum/class_3/event/p3_error_mean": 23.4354248046875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 13.728952407836914, "metrics/particle/momentum/class_3/event/px_bias": 0.8411746621131897, "metrics/particle/momentum/class_3/event/py_bias": -3.4130330085754395, "metrics/particle/momentum/class_3/event/pz_abs_mean": 16.08281707763672, "metrics/particle/momentum/class_3/event/pz_bias": -1.557036280632019, "metrics/particle/momentum/class_3/event/pz_residual_rms": 27.102554321289062, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04060446098446846, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19886387884616852, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19555599987506866, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.016498155891895294, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2284315824508667, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.023255813866853714, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.8856462240219116, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0371529720723629, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.52178955078125, "metrics/particle/momentum/class_4/accepted/samples": 172.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6371129155158997, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.10709285736084, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.1026649475097656, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9157376289367676, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3858520984649658, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.272492408752441, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03536977618932724, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.079622983932495, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.015786360949277878, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.310886859893799, "metrics/particle/momentum/class_4/assigned/samples": 311.0, "metrics/particle/momentum/class_4/efficiency": 0.47252747416496277, "metrics/particle/momentum/class_4/event/energy_bias": -2.186300754547119, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.267871856689453, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.3155455589294434, "metrics/particle/momentum/class_4/event/px_bias": -0.33900028467178345, "metrics/particle/momentum/class_4/event/py_bias": 0.5918959379196167, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.13543963432312, "metrics/particle/momentum/class_4/event/pz_bias": -0.06295352429151535, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.94719934463501, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04714047536253929, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2545873522758484, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2518790066242218, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.10440006852149963, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2881399691104889, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0476190485060215, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.347985029220581, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.5866892337799072, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.294404029846191, "metrics/particle/momentum/class_5/accepted/samples": 84.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.19326192140579224, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.469091534614563, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.46191534399986267, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.033828407526016235, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.24347825348377228, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8852665424346924, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08695652335882187, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.3796613216400146, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.9694238901138306, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.872464179992676, "metrics/particle/momentum/class_5/assigned/samples": 115.0, "metrics/particle/momentum/class_5/efficiency": 0.5283018946647644, "metrics/particle/momentum/class_5/event/energy_bias": -3.5864787101745605, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.88627815246582, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.8388500213623047, "metrics/particle/momentum/class_5/event/px_bias": -0.11003868281841278, "metrics/particle/momentum/class_5/event/py_bias": 0.7789859175682068, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.476107120513916, "metrics/particle/momentum/class_5/event/pz_bias": -0.405265212059021, "metrics/particle/momentum/class_5/event/pz_residual_rms": 7.897249698638916, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6875166893005371, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6938727498054504, "metrics/set_slot/angular_recall_all_dr0p10": 0.8188948631286621, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5873836278915405, "metrics/set_slot/soft_count_bias_mean": -3.2586257457733154}, "step_train_time": 4.5082995891571045, "step_valid_time": 12196.73468208313, "step_total_time": 12201.242981910706, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0829750368646989, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2995287899888741, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2949, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2525, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2336, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8341592335645854, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8562224482875551, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8450468540829986, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14377755171244488, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7717211760819292, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7921329264157342, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7817938420348058, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8341592335645854, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0364670606710218, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26886238685543185, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2204, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1891, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1826, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8359858532272325, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8579854809437386, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8468428123600538, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14201451905626133, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8072502210433244, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8284936479128857, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8177339901477833, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8359858532272325, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0197703958834892, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26561771240773524, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1238, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1039, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 993, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8066770186335404, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8392568659127625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8226444972288203, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16074313408723748, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7709627329192547, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8021001615508885, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7862232779097387, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8066770186335404} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_50000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..b668661f24a8816daee0572e331d7e40eade0ddd --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3687701345461436, "train": {"Classification_binary": 0.2148411124944687, "Cardinality": 0.17301788926124573, "Classification": 0.09286925941705704, "Regression_pt": 0.1643490046262741, "Regression_eta": 0.0016848936211317778, "Regression_sin_phi": 0.0009803669527173042, "Regression_cos_phi": 0.0009308651788160205, "Regression_energy": 0.16384662687778473, "HitGrouping": 0.23641154170036316, "Auxiliary": 0.29359832406044006, "Total": 1.3381712436676025, "Total_unweighted": 0.6395021291682497}, "train_group": {}, "valid": {"Classification_binary": 0.181916614373525, "Cardinality": 0.131487637758255, "Classification": 0.09093754490216573, "Regression_pt": 0.16662046313285828, "Regression_eta": 0.0016207882824043434, "Regression_sin_phi": 0.0009385498706251383, "Regression_cos_phi": 0.0009446618302414814, "Regression_energy": 0.16602768500645956, "HitGrouping": 0.23447845379511514, "Auxiliary": 0.26517895857493085, "Total": 1.2300923665364583, "Total_unweighted": 0.6090063073982795, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7315647602081299, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.35351327061653137, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.43402284383773804, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3442721664905548, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.354427695274353, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6094266772270203, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.662335991859436, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2994391620159149, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.42189088463783264, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.288718044757843, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.33423036336898804, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.7102677226066589, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7521766424179077, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3516962230205536, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4112178385257721, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.34331220388412476, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3383869230747223, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5706022381782532, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7374855279922485, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3475936949253082, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41951367259025574, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.33872416615486145, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3431759178638458, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5956230759620667, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025559065863490105, "metrics/hit_grouping/calorimeter/background_recall": 0.628694474697113, "metrics/hit_grouping/calorimeter/completeness": 0.3957969844341278, "metrics/hit_grouping/calorimeter/energy_completeness": 0.42146924138069153, "metrics/hit_grouping/calorimeter/energy_purity": 0.4164807200431824, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4966430366039276, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7568646669387817, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3612438142299652, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.35138848423957825, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3556523323059082, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3658914566040039, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5489400029182434, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7333075404167175, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3589818775653839, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.45972830057144165, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3464619815349579, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.353729248046875, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6079573035240173, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7008197903633118, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.30683332681655884, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5218051075935364, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.29482951760292053, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2840968668460846, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6815826296806335, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7237293720245361, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.37311768531799316, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5067580342292786, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3625348210334778, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3638347089290619, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6323886513710022, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7239044904708862, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.44855162501335144, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6268969178199768, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.43564003705978394, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.43566933274269104, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6148613095283508, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7540122270584106, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3072349429130554, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.29918307065963745, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.30072465538978577, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.30913063883781433, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5536360144615173, "metrics/hit_grouping/calorimeter/purity": 0.3898753821849823, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8517410755157471, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07262735813856125, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.16769230365753174, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0722142830491066, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.035527292639017105, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05538461357355118, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7601800560951233, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14211630821228027, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12428734451532364, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14050717651844025, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13625477254390717, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4201824367046356, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7337449193000793, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3652726113796234, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.44186124205589294, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.35580530762672424, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.36163443326950073, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6172285079956055, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7192123532295227, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44142258167266846, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.44096407294273376, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4461144506931305, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6171548366546631, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7737995386123657, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3527762293815613, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4744587540626526, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47561901807785034, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.475602924823761, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6739623546600342, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43750491738319397, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3919282555580139, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3974402844905853, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6802394390106201, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6966727375984192, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43160778284072876, "metrics/hit_grouping/tracker/all/object_completeness": 0.4143776595592499, "metrics/hit_grouping/tracker/all/object_purity": 0.419408917427063, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6430662870407104, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006765144411474466, "metrics/hit_grouping/tracker/background_recall": 0.039625659584999084, "metrics/hit_grouping/tracker/completeness": 0.43439966440200806, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.52424156665802, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7056098580360413, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42505693435668945, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4678899049758911, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47646042704582214, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6371753811836243, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6877834796905518, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44907671213150024, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3976331353187561, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.40469643473625183, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6596230864524841, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6862780451774597, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4275889992713928, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3133791387081146, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.30839431285858154, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6385517716407776, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7179005146026611, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.450754851102829, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43151310086250305, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4310038983821869, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6116103529930115, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7459479570388794, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44940081238746643, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4428096115589142, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4257882833480835, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5599201321601868, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6603538990020752, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40353548526763916, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38705533742904663, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40318191051483154, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.69866544008255, "metrics/hit_grouping/tracker/purity": 0.41483667492866516, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8871476650238037, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.017376193776726723, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.028027363121509552, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03136833757162094, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0803648978471756, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6737572550773621, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21358217298984528, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.21886186301708221, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.21874915063381195, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5728356838226318, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6733995079994202, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5127206444740295, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4891388416290283, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4949854612350464, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7275249361991882, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.44028639793395996, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5444902181625366, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6008347868919373, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.589045524597168, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.04544093832373619, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05824076384305954, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.88347053527832, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2876264452934265, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1571866273880005, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8617134690284729, "metrics/particle/class_1/pid_accuracy": 0.963284432888031, "metrics/particle/class_2/efficiency": 0.3958941102027893, "metrics/particle/class_2/pid_accuracy": 0.6183054447174072, "metrics/particle/class_3/efficiency": 0.7066054344177246, "metrics/particle/class_3/pid_accuracy": 0.9642651081085205, "metrics/particle/class_4/efficiency": 0.6958613991737366, "metrics/particle/class_4/pid_accuracy": 0.36652836203575134, "metrics/particle/class_5/efficiency": 0.8420000076293945, "metrics/particle/class_5/pid_accuracy": 0.40142518281936646, "metrics/particle/count/bias_mean": -3.705078125, "metrics/particle/count/mae": 4.874349117279053, "metrics/particle/count/prediction_mean": 66.35091400146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07632353156805038, "metrics/particle/event/energy_response_mean": 0.9757282137870789, "metrics/particle/event/met_abs_error": 7.830078125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0631970688700676, "metrics/particle/event/scalar_pt_response_mean": 0.9877040386199951, "metrics/particle/event/vector_pt_closure": 0.0635087862610817, "metrics/particle/matched/delta_eta_abs_mean": 0.014271722175180912, "metrics/particle/matched/delta_phi_abs_mean": 0.014596868306398392, "metrics/particle/matched/delta_r_mean": 0.02288844808936119, "metrics/particle/matched/energy_relative_abs_mean": 0.16050828993320465, "metrics/particle/matched/pid_accuracy": 0.9404516816139221, "metrics/particle/matched/pt_relative_abs_mean": 0.15970812737941742, "metrics/particle/matching/duplicate_fraction": 0.061011627316474915, "metrics/particle/matching/efficiency": 0.7542423009872437, "metrics/particle/matching/f1": 0.7747290134429932, "metrics/particle/matching/purity": 0.7963597178459167, "metrics/particle/matching/target_coverage_dr0p05": 0.7153039574623108, "metrics/particle/matching/target_coverage_dr0p10": 0.816097617149353, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5299146771430969, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7926602363586426, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022251395508646965, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15618851780891418, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1557137668132782, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009579061530530453, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1961459517478943, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008255298249423504, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7911357283592224, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.002504556905478239, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1109697818756104, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8116.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26529017090797424, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2551137208938599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4043599367141724, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2260758876800537, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15140075981616974, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.02224349975586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01927279867231846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2210233211517334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.046912260353565216, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3757975101470947, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10066.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7421360611915588, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.3882505893707275, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.0212984085083, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.411450386047363, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06343373656272888, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.11351299285888672, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.722270011901855, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.017229191958904266, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.428306579589844, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023017138242721558, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16731207072734833, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16548164188861847, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013256766833364964, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20597383379936218, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03720155358314514, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7285034656524658, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13551665842533112, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.172581672668457, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1801.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.44087234139442444, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1639842987060547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.43392276763916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2486588954925537, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20288066565990448, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.347463607788086, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06378600746393204, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.675565719604492, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.10213720798492432, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.876317501068115, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2430.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5961602330207825, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.465080738067627, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.452200889587402, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4671118259429932, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04999127984046936, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10612864792346954, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.270795822143555, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5947993993759155, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.163801193237305, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023059802129864693, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16092416644096375, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.160267174243927, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011997032910585403, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20099405944347382, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008336310856975615, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27098795771598816, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.003443838795647025, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8179875612258911, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16794.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22151494026184082, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3432778120040894, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.108542561531067, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9262983798980713, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15169082581996918, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.560088157653809, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005217391531914473, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5004074573516846, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005768814589828253, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0462698936462402, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20700.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7834118604660034, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.34906625747680664, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.978818893432617, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.387264251708984, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.17946799099445343, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.1151984930038452, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.531056880950928, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.26451656222343445, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.141139507293701, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022811297327280045, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1599159687757492, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15923523902893066, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011347283609211445, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19987556338310242, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005540788639336824, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5273057222366333, "metrics/particle/momentum/all/accepted/pz_bias": -0.006211024709045887, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7171437740325928, "metrics/particle/momentum/all/accepted/samples": 26711.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25084617733955383, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3034192323684692, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2952630519866943, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.113998532295227, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15535004436969757, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.011198997497559, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013766719028353691, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8781444430351257, "metrics/particle/momentum/all/assigned/pz_bias": 0.010345821268856525, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0867247581481934, "metrics/particle/momentum/all/assigned/samples": 33196.0, "metrics/particle/momentum/all/efficiency": 0.7546759247779846, "metrics/particle/momentum/all/event/energy_bias": -8.202398300170898, "metrics/particle/momentum/all/event/p3_error_mean": 21.845767974853516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.862820625305176, "metrics/particle/momentum/all/event/px_bias": 0.19291046261787415, "metrics/particle/momentum/all/event/py_bias": 1.1225829124450684, "metrics/particle/momentum/all/event/pz_abs_mean": 16.265457153320312, "metrics/particle/momentum/all/event/pz_bias": -1.347512125968933, "metrics/particle/momentum/all/event/pz_residual_rms": 25.135534286499023, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0238327756524086, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15182095766067505, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15132209658622742, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012702438049018383, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1910136193037033, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005975957028567791, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5704490542411804, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.008006819523870945, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7485848665237427, "metrics/particle/momentum/class_1/accepted/samples": 14391.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11824464052915573, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3931666314601898, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.44725531339645386, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.29085832834243774, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08721573650836945, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.391693115234375, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009711124934256077, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7278653383255005, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.010740167461335659, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.263138771057129, "metrics/particle/momentum/class_1/assigned/samples": 16270.0, "metrics/particle/momentum/class_1/efficiency": 0.8624078631401062, "metrics/particle/momentum/class_1/event/energy_bias": 4.251537322998047, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.889455795288086, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.577425003051758, "metrics/particle/momentum/class_1/event/px_bias": -0.5225438475608826, "metrics/particle/momentum/class_1/event/py_bias": 0.071574367582798, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.891870498657227, "metrics/particle/momentum/class_1/event/pz_bias": -1.065126895904541, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.788646697998047, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03212863951921463, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19992780685424805, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2015591412782669, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0272278543561697, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24102914333343506, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03159692510962486, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8504443168640137, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04272692650556564, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.349368095397949, "metrics/particle/momentum/class_2/accepted/samples": 1171.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7192514538764954, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8716690540313721, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9383765459060669, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5174885988235474, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34276285767555237, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.172622203826904, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05846298858523369, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9880177974700928, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.005630730651319027, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.644440174102783, "metrics/particle/momentum/class_2/assigned/samples": 2121.0, "metrics/particle/momentum/class_2/efficiency": 0.3933490216732025, "metrics/particle/momentum/class_2/event/energy_bias": -12.715126037597656, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.41222381591797, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.115362167358398, "metrics/particle/momentum/class_2/event/px_bias": 1.149749517440796, "metrics/particle/momentum/class_2/event/py_bias": -0.3868815302848816, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.137308120727539, "metrics/particle/momentum/class_2/event/pz_bias": 0.15126335620880127, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.16529655456543, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02050764299929142, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1658969521522522, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1648806780576706, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013253772631287575, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20600956678390503, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019507664255797863, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2990269362926483, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.014838733710348606, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9550439715385437, "metrics/particle/momentum/class_3/accepted/samples": 10765.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33289340138435364, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.400470733642578, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3161823749542236, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.141456127166748, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2032855600118637, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.051170349121094, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011394617147743702, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7107372879981995, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.009580112993717194, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.664090156555176, "metrics/particle/momentum/class_3/assigned/samples": 14305.0, "metrics/particle/momentum/class_3/efficiency": 0.7078976631164551, "metrics/particle/momentum/class_3/event/energy_bias": 2.90963077545166, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.176867485046387, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.165684223175049, "metrics/particle/momentum/class_3/event/px_bias": -0.600816547870636, "metrics/particle/momentum/class_3/event/py_bias": 0.9443244934082031, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.4063081741333, "metrics/particle/momentum/class_3/event/pz_bias": -1.042177438735962, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.059364318847656, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02314775623381138, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1737171858549118, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16561278700828552, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0539134182035923, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20749275386333466, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007751937955617905, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9248308539390564, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.29883983731269836, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1042933464050293, "metrics/particle/momentum/class_4/accepted/samples": 258.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.31461504101753235, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8633078336715698, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6112339496612549, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4864590167999268, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21802325546741486, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.484508037567139, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00872093066573143, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1001436710357666, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3526328206062317, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1834657192230225, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.708791196346283, "metrics/particle/momentum/class_4/event/energy_bias": -1.498083472251892, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.060744285583496, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1331231594085693, "metrics/particle/momentum/class_4/event/px_bias": 0.17820215225219727, "metrics/particle/momentum/class_4/event/py_bias": 0.15713009238243103, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8138126134872437, "metrics/particle/momentum/class_4/event/pz_bias": 0.6568414568901062, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.865664482116699, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015678687021136284, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17337097227573395, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17430181801319122, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03497016802430153, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2188514620065689, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.01587301678955555, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9923123121261597, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.14342468976974487, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.6135752201080322, "metrics/particle/momentum/class_5/accepted/samples": 126.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04775587096810341, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2753908336162567, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.27667251229286194, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03566036373376846, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1794871836900711, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3932579457759857, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.726787567138672, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4981350600719452, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.060802936553955, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.7924528121948242, "metrics/particle/momentum/class_5/event/energy_bias": -1.150357723236084, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.959296703338623, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.199181079864502, "metrics/particle/momentum/class_5/event/px_bias": -0.011680863797664642, "metrics/particle/momentum/class_5/event/py_bias": 0.33643537759780884, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.625767469406128, "metrics/particle/momentum/class_5/event/pz_bias": -0.04831264168024063, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.078410625457764, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.40120449662208557, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.40244215726852417, "metrics/set_slot/angular_recall_all_dr0p10": 0.9458022713661194, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8082448840141296, "metrics/set_slot/soft_count_bias_mean": -0.4046984910964966}, "step_train_time": 5.501085042953491, "step_valid_time": 37657.167838811874, "step_total_time": 37662.668924331665, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008956044514727, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16748119833408026, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2850, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2805, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9415262636273538, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.95, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457441513190642, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9266600594648167, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.935, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9308113489298159, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9415262636273538, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9912606980621184, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15205777663173825, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2104, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2092, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9301503094606542, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.942230183609494, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9361512791991101, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057769816390506046, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9248452696728559, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9368562472010747, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9308120133481647, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9301503094606542, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9935263827952737, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15430930689636602, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1187, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.921583850931677, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9287949921752738, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9251753702260327, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07120500782472614, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9184782608695652, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9256651017214398, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9220576773187841, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.921583850931677} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_6000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..32382d83e08c9da5dbb70f7c9a64ca0ea8980fb8 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28425241614553726, "train": {"Classification_binary": 0.35896503925323486, "Cardinality": 0.2724640965461731, "Classification": 0.2578643560409546, "Regression_pt": 0.3931463360786438, "Regression_eta": 0.0047163269482553005, "Regression_sin_phi": 0.002560423919931054, "Regression_cos_phi": 0.0023165205493569374, "Regression_energy": 0.3919292688369751, "HitGrouping": 0.27708643674850464, "Auxiliary": 0.4815443456172943, "Total": 2.463334560394287, "Total_unweighted": 1.4114982716273516}, "train_group": {}, "valid": {"Classification_binary": 0.3304656942685445, "Cardinality": 0.22039560476938883, "Classification": 0.23923641443252563, "Regression_pt": 0.3868595361709595, "Regression_eta": 0.004441224038600922, "Regression_sin_phi": 0.002471659332513809, "Regression_cos_phi": 0.0024622847946981588, "Regression_energy": 0.3842404286066691, "HitGrouping": 0.27398882309595746, "Auxiliary": 0.44945422808329266, "Total": 2.324739138285319, "Total_unweighted": 1.3501772416445117, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7393897771835327, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.20037654042243958, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4011523425579071, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.19194108247756958, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.17358142137527466, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6239727735519409, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7227909564971924, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.18485763669013977, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34090137481689453, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.17449462413787842, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.15666215121746063, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6187732815742493, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7454166412353516, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.20231866836547852, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.36907345056533813, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.19316153228282928, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.17980864644050598, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6070137619972229, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7414478063583374, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2001275271177292, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3768692910671234, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.19109445810317993, "metrics/hit_grouping/calorimeter/all/object_purity": 0.17572860419750214, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6135258078575134, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025474632158875465, "metrics/hit_grouping/calorimeter/background_recall": 0.6244378685951233, "metrics/hit_grouping/calorimeter/completeness": 0.2816622257232666, "metrics/hit_grouping/calorimeter/energy_completeness": 0.32197344303131104, "metrics/hit_grouping/calorimeter/energy_purity": 0.31833696365356445, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.6225430965423584, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.763150155544281, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.19917626678943634, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3200955390930176, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.193041130900383, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.1827753782272339, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5661391615867615, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7297542691230774, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2144344598054886, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4147258698940277, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.202200248837471, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.186055526137352, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6390101909637451, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7076969146728516, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.1872318983078003, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4575906991958618, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.1754007488489151, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.15005135536193848, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6872708797454834, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7094733715057373, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2298470288515091, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.478168785572052, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.21836626529693604, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.19608058035373688, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6794145107269287, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.727242648601532, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.37465232610702515, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.66300368309021, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.35841843485832214, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.34187984466552734, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6300366520881653, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7776400446891785, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.14584442973136902, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23135128617286682, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.14036424458026886, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.1325877606868744, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5405785441398621, "metrics/hit_grouping/calorimeter/purity": 0.2774241864681244, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8955317735671997, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04689748212695122, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1569230705499649, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04497436061501503, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.004273888189345598, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8130223751068115, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.055662743747234344, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09920182079076767, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.05388106033205986, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.04073556140065193, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3797035217285156, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7340469360351562, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.21170252561569214, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3975275456905365, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.20210084319114685, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.18714727461338043, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6391329765319824, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6950769424438477, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.45606693625450134, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.31968361139297485, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.27060654759407043, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.675582766532898, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7724443078041077, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.39203590154647827, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.36764371395111084, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2794050872325897, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4941110610961914, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6326072812080383, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4502638280391693, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.28300201892852783, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.25134679675102234, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7360793948173523, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6641289591789246, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4471943974494934, "metrics/hit_grouping/tracker/all/object_completeness": 0.3017243444919586, "metrics/hit_grouping/tracker/all/object_purity": 0.2597973048686981, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6965804100036621, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00464226771146059, "metrics/hit_grouping/tracker/background_recall": 0.019013209268450737, "metrics/hit_grouping/tracker/completeness": 0.31847357749938965, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.648556649684906, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6589679718017578, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4584965705871582, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.35713687539100647, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.32811638712882996, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7107972502708435, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6627967357635498, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45821434259414673, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.27856481075286865, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.2261684238910675, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6950314044952393, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6770011782646179, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41039642691612244, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2033231258392334, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.14386899769306183, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6666666865348816, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6968019604682922, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48014017939567566, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3166007995605469, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.24906505644321442, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6587886214256287, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7176639437675476, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4533954858779907, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3051830232143402, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2514786124229431, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6125166416168213, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.612151026725769, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40318426489830017, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2817360460758209, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2752413749694824, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7605947256088257, "metrics/hit_grouping/tracker/purity": 0.30348262190818787, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8972550630569458, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.010425716638565063, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.01345672458410263, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.011113470420241356, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06950477510690689, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6827740669250488, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.13634006679058075, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.09718889743089676, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.07319939136505127, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5785381197929382, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6303279399871826, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5419396758079529, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.364184707403183, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.31483641266822815, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.795230507850647, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3573976457118988, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6749999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5270935297012329, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5764445066452026, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5352433919906616, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.12832967936992645, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0044386181980371475, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.378633499145508, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2210681289434433, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.09062831103801727, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5753462314605713, "metrics/particle/class_1/pid_accuracy": 0.8868560194969177, "metrics/particle/class_2/efficiency": 0.3253309428691864, "metrics/particle/class_2/pid_accuracy": 0.170689657330513, "metrics/particle/class_3/efficiency": 0.47720593214035034, "metrics/particle/class_3/pid_accuracy": 0.9021309018135071, "metrics/particle/class_4/efficiency": 0.5004812479019165, "metrics/particle/class_4/pid_accuracy": 0.21346153318881989, "metrics/particle/class_5/efficiency": 0.5479999780654907, "metrics/particle/class_5/pid_accuracy": 0.24087591469287872, "metrics/particle/count/bias_mean": -7.916666507720947, "metrics/particle/count/mae": 9.143229484558105, "metrics/particle/count/prediction_mean": 62.13932418823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1259443610906601, "metrics/particle/event/energy_response_mean": 0.967671811580658, "metrics/particle/event/met_abs_error": 13.760354042053223, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1126234233379364, "metrics/particle/event/scalar_pt_response_mean": 1.0117322206497192, "metrics/particle/event/vector_pt_closure": 0.1025991141796112, "metrics/particle/matched/delta_eta_abs_mean": 0.02109505794942379, "metrics/particle/matched/delta_phi_abs_mean": 0.02197257988154888, "metrics/particle/matched/delta_r_mean": 0.03399425372481346, "metrics/particle/matched/energy_relative_abs_mean": 0.20089226961135864, "metrics/particle/matched/pid_accuracy": 0.8456994295120239, "metrics/particle/matched/pt_relative_abs_mean": 0.19744528830051422, "metrics/particle/matching/duplicate_fraction": 0.11144521832466125, "metrics/particle/matching/efficiency": 0.5112726092338562, "metrics/particle/matching/f1": 0.5418907403945923, "metrics/particle/matching/purity": 0.5764096975326538, "metrics/particle/matching/target_coverage_dr0p05": 0.4522145688533783, "metrics/particle/matching/target_coverage_dr0p10": 0.608497679233551, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.31412747502326965, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5781183242797852, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03283914923667908, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20068515837192535, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19805176556110382, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.022393979132175446, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23980028927326202, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01881520077586174, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3338556289672852, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.11801013350486755, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.054860830307007, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5368.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5952916741371155, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.481696367263794, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.105297803878784, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.891892433166504, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.33203819394111633, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.023277282714844, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03530195355415344, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.0191433429718018, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.031300585716962814, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.424895763397217, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9008.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4908559024333954, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1932566165924072, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 14.069364547729492, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.66310977935791, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.9372482895851135, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.12704601883888245, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.821857452392578, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3637797236442566, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.565227508544922, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03151979297399521, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20747491717338562, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.20141664147377014, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.026452457532286644, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24440163373947144, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.05020575970411301, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.3888347148895264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11085687577724457, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.271286964416504, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1215.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7525655031204224, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7776706218719482, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.030105113983154, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.816110610961914, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3454457223415375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 26.40083122253418, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07364340871572495, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2626044750213623, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.008768118917942047, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.82614803314209, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2064.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.40218469500541687, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -10.65942096710205, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.814372062683105, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.0233876705169678, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.14879140257835388, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1519821435213089, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.586809158325195, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2159704864025116, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.197389602661133, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.034687623381614685, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19953088462352753, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19665011763572693, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.007176951505243778, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23798082768917084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0021622555796056986, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.45046913623809814, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.025797855108976364, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1783212423324585, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11562.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.549818754196167, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.5959413051605225, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.956753730773926, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.7617030143737793, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.34392037987709045, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 15.800333023071289, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.013954884372651577, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0616450309753418, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02514922432601452, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.4156811237335205, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19993.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5393478274345398, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 7.92466926574707, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.17795181274414, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.382648468017578, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.001215696334839, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3991677761077881, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.800145626068115, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.0562625527381897, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.205745697021484, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.033928655087947845, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.2004043012857437, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19738395512104034, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.012969440780580044, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23895491659641266, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.01030586939305067, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8416035175323486, "metrics/particle/momentum/all/accepted/pz_bias": -0.058773454278707504, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.346515655517578, "metrics/particle/momentum/all/accepted/samples": 18145.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5764754414558411, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.1520326137542725, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.1375834941864014, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.9359519481658936, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3405762016773224, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.05910873413086, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.024110736325383186, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4855279922485352, "metrics/particle/momentum/all/assigned/pz_bias": 0.024679433554410934, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.0351128578186035, "metrics/particle/momentum/all/assigned/samples": 31065.0, "metrics/particle/momentum/all/efficiency": 0.5126575231552124, "metrics/particle/momentum/all/event/energy_bias": -3.9280083179473877, "metrics/particle/momentum/all/event/p3_error_mean": 33.9759521484375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.522052764892578, "metrics/particle/momentum/all/event/px_bias": 4.087255954742432, "metrics/particle/momentum/all/event/py_bias": 0.42410382628440857, "metrics/particle/momentum/all/event/pz_abs_mean": 24.79136085510254, "metrics/particle/momentum/all/event/pz_bias": 0.5234876871109009, "metrics/particle/momentum/all/event/pz_residual_rms": 36.542633056640625, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.040495194494724274, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21814952790737152, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21660421788692474, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03087833896279335, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.25598499178886414, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.01278746034950018, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9929419159889221, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.09958866983652115, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.4724056720733643, "metrics/particle/momentum/class_1/accepted/samples": 9697.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.390036016702652, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.3260120153427124, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.5081819295883179, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.2749541997909546, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2922479510307312, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.94180965423584, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01980784349143505, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.3623847961425781, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.011484479531645775, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.4539966583251953, "metrics/particle/momentum/class_1/assigned/samples": 15196.0, "metrics/particle/momentum/class_1/efficiency": 0.5811110734939575, "metrics/particle/momentum/class_1/event/energy_bias": 12.037694931030273, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.423866271972656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.85676383972168, "metrics/particle/momentum/class_1/event/px_bias": 5.671351432800293, "metrics/particle/momentum/class_1/event/py_bias": 0.1416195034980774, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.0513916015625, "metrics/particle/momentum/class_1/event/pz_bias": 0.6375695466995239, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.370983123779297, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.046032048761844635, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2279413640499115, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22598779201507568, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05894779413938522, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.267236590385437, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03789473697543144, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0119194984436035, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10471107810735703, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.674160480499268, "metrics/particle/momentum/class_2/accepted/samples": 950.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7964513301849365, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8593481779098511, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2375006675720215, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8258429169654846, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3507501184940338, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.949185371398926, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06518365442752838, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.95467209815979, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07308560609817505, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.384899616241455, "metrics/particle/momentum/class_2/assigned/samples": 1933.0, "metrics/particle/momentum/class_2/efficiency": 0.31911319494247437, "metrics/particle/momentum/class_2/event/energy_bias": -27.227378845214844, "metrics/particle/momentum/class_2/event/p3_error_mean": 29.470077514648438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.229290008544922, "metrics/particle/momentum/class_2/event/px_bias": -0.05708283185958862, "metrics/particle/momentum/class_2/event/py_bias": 0.2916261851787567, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.373960494995117, "metrics/particle/momentum/class_2/event/pz_bias": -0.8705284595489502, "metrics/particle/momentum/class_2/event/pz_residual_rms": 32.436798095703125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023287834599614143, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17234882712364197, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1671704202890396, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.018844306468963623, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2087632417678833, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0026312144473195076, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.443926602602005, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004933672491461039, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.5195564031600952, "metrics/particle/momentum/class_3/accepted/samples": 7221.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7580456733703613, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.546520233154297, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.259981155395508, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.126394271850586, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.39385393261909485, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.286273956298828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02197149582207203, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.3774603605270386, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.059846457093954086, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.118744373321533, "metrics/particle/momentum/class_3/assigned/samples": 13472.0, "metrics/particle/momentum/class_3/efficiency": 0.47484710812568665, "metrics/particle/momentum/class_3/event/energy_bias": 15.950005531311035, "metrics/particle/momentum/class_3/event/p3_error_mean": 21.39874267578125, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 12.337257385253906, "metrics/particle/momentum/class_3/event/px_bias": -1.241325855255127, "metrics/particle/momentum/class_3/event/py_bias": -0.18787576258182526, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.744671821594238, "metrics/particle/momentum/class_3/event/pz_bias": -0.07914689183235168, "metrics/particle/momentum/class_3/event/pz_residual_rms": 24.31251335144043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.041668277233839035, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18835753202438354, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18964380025863647, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.014694726094603539, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23166009783744812, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.033149171620607376, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.7259057760238647, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4070107042789459, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.202333927154541, "metrics/particle/momentum/class_4/accepted/samples": 181.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6119390726089478, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.985629081726074, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.72914719581604, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.5544896125793457, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.37195122241973877, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.294045448303223, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.8546454906463623, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.11717426776885986, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.3105034828186035, "metrics/particle/momentum/class_4/assigned/samples": 328.0, "metrics/particle/momentum/class_4/efficiency": 0.4972527325153351, "metrics/particle/momentum/class_4/event/energy_bias": -2.294269561767578, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.149752140045166, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.06463885307312, "metrics/particle/momentum/class_4/event/px_bias": -0.6345756649971008, "metrics/particle/momentum/class_4/event/py_bias": 0.12266596406698227, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.2157769203186035, "metrics/particle/momentum/class_4/event/pz_bias": 0.4801405668258667, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.293872356414795, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.036663126200437546, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2684600055217743, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.26009631156921387, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.1387278288602829, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2953977882862091, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02083333395421505, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2190752029418945, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.4093792140483856, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.747096538543701, "metrics/particle/momentum/class_5/accepted/samples": 96.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.21009601652622223, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5638744831085205, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5366286039352417, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04510471597313881, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.24264705181121826, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.602910041809082, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10294117778539658, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.178529262542725, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3292219638824463, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.025379180908203, "metrics/particle/momentum/class_5/assigned/samples": 136.0, "metrics/particle/momentum/class_5/efficiency": 0.6037735939025879, "metrics/particle/momentum/class_5/event/energy_bias": -2.394059181213379, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.900104522705078, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.8229079246520996, "metrics/particle/momentum/class_5/event/px_bias": 0.348889023065567, "metrics/particle/momentum/class_5/event/py_bias": 0.05606798827648163, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.540231943130493, "metrics/particle/momentum/class_5/event/pz_bias": 0.35545313358306885, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.216813087463379, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.653225839138031, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.656254231929779, "metrics/set_slot/angular_recall_all_dr0p10": 0.8347954750061035, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6311915516853333, "metrics/set_slot/soft_count_bias_mean": 1.0007472038269043}, "step_train_time": 7.179805755615234, "step_valid_time": 14766.680802822113, "step_total_time": 14773.860608816147, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0786969634511347, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.301918359299129, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2945, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2598, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2432, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8582755203171457, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8821731748726656, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.870060281312793, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11782682512733446, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8034357449620085, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8258064516129032, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8144675150703282, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8582755203171457, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0103003031646294, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.250503153099513, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2199, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1932, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1878, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8541114058355438, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.878581173260573, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8661735036987223, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12141882673942701, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.830238726790451, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8540245566166439, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8419636852723604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8541114058355438, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211569673185799, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2586080391517811, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1233, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1046, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1013, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.812111801242236, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8483373884833739, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8298294327647758, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1516626115166261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7864906832298136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8215733982157339, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8036493454978183, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.812111801242236} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_7000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..e64980f1a6a96ce941c1872f5703e50a82982739 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.3316278188364601, "train": {"Classification_binary": 0.37543678283691406, "Cardinality": 0.26407426595687866, "Classification": 0.24903810024261475, "Regression_pt": 0.3842272162437439, "Regression_eta": 0.005140745081007481, "Regression_sin_phi": 0.00296831876039505, "Regression_cos_phi": 0.003013370092958212, "Regression_energy": 0.38194701075553894, "HitGrouping": 0.273666650056839, "Auxiliary": 0.4695436358451843, "Total": 2.5406947135925293, "Total_unweighted": 1.4017715440131724}, "train_group": {}, "valid": {"Classification_binary": 0.3235652446746826, "Cardinality": 0.2117281754811605, "Classification": 0.22147715091705322, "Regression_pt": 0.3666567802429199, "Regression_eta": 0.004509368600944678, "Regression_sin_phi": 0.0023890569185217223, "Regression_cos_phi": 0.0023552452524503074, "Regression_energy": 0.3636555274327596, "HitGrouping": 0.27001450459162396, "Auxiliary": 0.4267045259475708, "Total": 2.2396872838338218, "Total_unweighted": 1.2846083740393321, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7306022644042969, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.21061240136623383, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3997355103492737, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.20040783286094666, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.18901298940181732, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6354019045829773, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7064338326454163, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.19062615931034088, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3700440526008606, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.17772963643074036, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.16473880410194397, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6513046622276306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7332899570465088, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2116129845380783, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3715525269508362, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.20173805952072144, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.20015418529510498, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6361945867538452, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7300162315368652, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.20940856635570526, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.38048630952835083, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.19915653765201569, "metrics/hit_grouping/calorimeter/all/object_purity": 0.19339239597320557, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6372948288917542, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0253708828240633, "metrics/hit_grouping/calorimeter/background_recall": 0.6244008541107178, "metrics/hit_grouping/calorimeter/completeness": 0.29424339532852173, "metrics/hit_grouping/calorimeter/energy_completeness": 0.33970531821250916, "metrics/hit_grouping/calorimeter/energy_purity": 0.33580508828163147, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.6130492091178894, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7489523887634277, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.20269328355789185, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.31770676374435425, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.19541646540164948, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.19703806936740875, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.594207227230072, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7149069309234619, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.229289710521698, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.42503640055656433, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.215186208486557, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.21089862287044525, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.667515754699707, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.705679714679718, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2029045671224594, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46694475412368774, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.19036871194839478, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.16743259131908417, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6970041990280151, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7038247585296631, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.23964516818523407, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48632824420928955, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.22706270217895508, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2161587029695511, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6954219937324524, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7332789897918701, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4053214490413666, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6797488331794739, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3911779522895813, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.34269455075263977, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6211407780647278, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.757756233215332, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.15183264017105103, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22840498387813568, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.14458401501178741, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.1498119980096817, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5768715739250183, "metrics/hit_grouping/calorimeter/purity": 0.2897852063179016, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8825708031654358, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.038822680711746216, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14769230782985687, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.03700000047683716, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012157808057963848, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7989832758903503, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.053538549691438675, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09692132472991943, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.0527803860604763, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.038870010524988174, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.4002280533313751, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7227979302406311, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.22200943529605865, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4017575979232788, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2110317200422287, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.20614275336265564, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.663529634475708, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6976537108421326, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44172146916389465, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.33163848519325256, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.28844520449638367, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6657202839851379, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7713503241539001, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3830622434616089, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.37086623907089233, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.30090489983558655, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4862591028213501, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6312641501426697, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4444356858730316, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.29366570711135864, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.27479544281959534, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7305662631988525, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6640457510948181, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4384092092514038, "metrics/hit_grouping/tracker/all/object_completeness": 0.31216952204704285, "metrics/hit_grouping/tracker/all/object_purity": 0.2813086211681366, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6894955635070801, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008332901634275913, "metrics/hit_grouping/tracker/background_recall": 0.030267132446169853, "metrics/hit_grouping/tracker/completeness": 0.3326001465320587, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6193666458129883, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6593829393386841, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.44018223881721497, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3643111288547516, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3518814742565155, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7037813067436218, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6651169061660767, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4551684856414795, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.29341721534729004, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.24521131813526154, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.68893963098526, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6732585430145264, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4166666567325592, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.21634647250175476, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.1629999876022339, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6583737730979919, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6972053050994873, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4791516959667206, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3302907645702362, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2708520293235779, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6504313349723816, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7117926478385925, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.474700391292572, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3266955018043518, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.27101975679397583, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6111850738525391, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6124518513679504, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3789510726928711, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.28600800037384033, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.29673999547958374, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7541559338569641, "metrics/hit_grouping/tracker/purity": 0.31812378764152527, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.879506528377533, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.010425716638565063, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.012270214036107063, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.012194112874567509, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07124239951372147, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6702547073364258, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.13374805450439453, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.10191141068935394, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.08978205919265747, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5816485285758972, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6340612173080444, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5312555432319641, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3768619894981384, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3396851122379303, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.785785973072052, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.3730483651161194, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5021452903747559, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.546247124671936, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5398793816566467, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11267895251512527, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.009074576199054718, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.158889770507812, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2368796169757843, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.10643978416919708, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5726948976516724, "metrics/particle/class_1/pid_accuracy": 0.9134190082550049, "metrics/particle/class_2/efficiency": 0.3157953917980194, "metrics/particle/class_2/pid_accuracy": 0.1637655347585678, "metrics/particle/class_3/efficiency": 0.4760259985923767, "metrics/particle/class_3/pid_accuracy": 0.8999053835868835, "metrics/particle/class_4/efficiency": 0.504331111907959, "metrics/particle/class_4/pid_accuracy": 0.20610687136650085, "metrics/particle/class_5/efficiency": 0.6079999804496765, "metrics/particle/class_5/pid_accuracy": 0.28947368264198303, "metrics/particle/count/bias_mean": -9.877604484558105, "metrics/particle/count/mae": 10.442708015441895, "metrics/particle/count/prediction_mean": 60.17838668823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.11113537102937698, "metrics/particle/event/energy_response_mean": 0.9696180820465088, "metrics/particle/event/met_abs_error": 12.030550956726074, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09062280505895615, "metrics/particle/event/scalar_pt_response_mean": 0.9898268580436707, "metrics/particle/event/vector_pt_closure": 0.09414414316415787, "metrics/particle/matched/delta_eta_abs_mean": 0.021285684779286385, "metrics/particle/matched/delta_phi_abs_mean": 0.020468328148126602, "metrics/particle/matched/delta_r_mean": 0.03291754797101021, "metrics/particle/matched/energy_relative_abs_mean": 0.20009921491146088, "metrics/particle/matched/pid_accuracy": 0.8591901659965515, "metrics/particle/matched/pt_relative_abs_mean": 0.19452306628227234, "metrics/particle/matching/duplicate_fraction": 0.09963866323232651, "metrics/particle/matching/efficiency": 0.5090422630310059, "metrics/particle/matching/f1": 0.5476504564285278, "metrics/particle/matching/purity": 0.5925958156585693, "metrics/particle/matching/target_coverage_dr0p05": 0.4642491936683655, "metrics/particle/matching/target_coverage_dr0p10": 0.6119082570075989, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3138858377933502, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5726167559623718, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03118741884827614, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19356106221675873, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19167329370975494, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009194347076117992, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2338210940361023, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.017264027148485184, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.2698065042495728, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.045191943645477295, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.9367003440856934, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5329.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5211149454116821, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.2821638584136963, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.383768081665039, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.1891770362854004, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3175683617591858, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.532844543457031, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.030453339219093323, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8951796293258667, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.18938244879245758, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.324553966522217, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8669.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.487289696931839, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.3976545333862305, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.969653129577637, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.089083671569824, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4665827751159668, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6106159687042236, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.054495811462402, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.5322365760803223, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.462921142578125, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02942466363310814, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19478286802768707, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1893884837627411, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010941801592707634, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23349256813526154, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04320000112056732, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0951876640319824, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1875433772802353, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.559177875518799, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1250.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6408815979957581, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4185326099395752, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.6133859157562256, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.4114975929260254, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3257281482219696, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.037540435791016, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06844659894704819, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.3140647411346436, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07545013725757599, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.696860313415527, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2060.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.41377028822898865, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.3593168258667, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.460639953613281, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.2015175819396973, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.003826737403869629, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.035949934273958206, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 14.210150718688965, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.660526156425476, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 32.085121154785156, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.033841922879219055, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20284293591976166, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1963442862033844, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0011032470501959324, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23819060623645782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00218150089494884, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4474719166755676, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.05656057223677635, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.185500144958496, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11460.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5190320611000061, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.1051723957061768, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.486853837966919, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.296384572982788, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3391687273979187, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.865394592285156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.013104013167321682, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9891118407249451, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06064236909151077, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.155067205429077, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19536.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5345897078514099, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.285767078399658, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.4631986618042, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.615004539489746, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.5698214769363403, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.8053191900253296, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.583139419555664, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.6220595836639404, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.490788459777832, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.032751649618148804, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19954241812229156, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19448241591453552, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.00417524017393589, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2365838587284088, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0094794612377882, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8045796155929565, "metrics/particle/momentum/all/accepted/pz_bias": 0.0362870879471302, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.209282159805298, "metrics/particle/momentum/all/accepted/samples": 18039.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5279224514961243, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.7546308040618896, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.534003973007202, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.3415770530700684, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.33206674456596375, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.272943496704102, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02184041030704975, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4068915843963623, "metrics/particle/momentum/all/assigned/pz_bias": 0.08825503289699554, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.114681243896484, "metrics/particle/momentum/all/assigned/samples": 30265.0, "metrics/particle/momentum/all/efficiency": 0.5096626281738281, "metrics/particle/momentum/all/event/energy_bias": -6.47120475769043, "metrics/particle/momentum/all/event/p3_error_mean": 32.71039581298828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.585897445678711, "metrics/particle/momentum/all/event/px_bias": 2.0325772762298584, "metrics/particle/momentum/all/event/py_bias": 2.4518849849700928, "metrics/particle/momentum/all/event/pz_abs_mean": 25.282730102539062, "metrics/particle/momentum/all/event/pz_bias": 3.4937705993652344, "metrics/particle/momentum/all/event/pz_residual_rms": 37.62028884887695, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0382259376347065, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21046561002731323, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21162615716457367, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.002322702668607235, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2522903084754944, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.010806317441165447, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9380641579627991, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.031156005337834358, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.354235887527466, "metrics/particle/momentum/class_1/accepted/samples": 9624.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.34102603793144226, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.2239447832107544, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.3564194440841675, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.154119610786438, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.29632338881492615, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.021851539611816, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.018283359706401825, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.309025764465332, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.026737695559859276, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.580871820449829, "metrics/particle/momentum/class_1/assigned/samples": 15041.0, "metrics/particle/momentum/class_1/efficiency": 0.5767363905906677, "metrics/particle/momentum/class_1/event/energy_bias": 21.8255558013916, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.932315826416016, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.994962692260742, "metrics/particle/momentum/class_1/event/px_bias": 0.6955419778823853, "metrics/particle/momentum/class_1/event/py_bias": 0.9749271869659424, "metrics/particle/momentum/class_1/event/pz_abs_mean": 19.451364517211914, "metrics/particle/momentum/class_1/event/pz_bias": -0.15999317169189453, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.915159225463867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.044948890805244446, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23041552305221558, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2253277599811554, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07189822942018509, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26500633358955383, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04266958311200142, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.096761703491211, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05576692894101143, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.736199378967285, "metrics/particle/momentum/class_2/accepted/samples": 914.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7378425598144531, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6995510458946228, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7977992296218872, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3799060583114624, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34398698806762695, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.4175498485565186, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06392199546098709, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.951589822769165, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.35404086112976074, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.460585594177246, "metrics/particle/momentum/class_2/assigned/samples": 1846.0, "metrics/particle/momentum/class_2/efficiency": 0.30702048540115356, "metrics/particle/momentum/class_2/event/energy_bias": -31.451677322387695, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.945661544799805, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.651954650878906, "metrics/particle/momentum/class_2/event/px_bias": 1.9367655515670776, "metrics/particle/momentum/class_2/event/py_bias": 0.7842110395431519, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.240236282348633, "metrics/particle/momentum/class_2/event/pz_bias": 0.12887001037597656, "metrics/particle/momentum/class_2/event/pz_residual_rms": 32.459957122802734, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023644516244530678, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18053390085697174, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16698090732097626, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024508139118552208, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2085367888212204, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002494111191481352, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.41677379608154297, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.04260445758700371, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1926180124282837, "metrics/particle/momentum/class_3/accepted/samples": 7217.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7188482284545898, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.799848556518555, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.133312225341797, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.987070083618164, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.37248530983924866, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.86128044128418, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018956979736685753, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.266196846961975, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.10663172602653503, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.231430530548096, "metrics/particle/momentum/class_3/assigned/samples": 12924.0, "metrics/particle/momentum/class_3/efficiency": 0.47458407282829285, "metrics/particle/momentum/class_3/event/energy_bias": 8.080090522766113, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.61318588256836, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.317693710327148, "metrics/particle/momentum/class_3/event/px_bias": 0.023464687168598175, "metrics/particle/momentum/class_3/event/py_bias": 0.2870578467845917, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.732632637023926, "metrics/particle/momentum/class_3/event/pz_bias": 2.8743038177490234, "metrics/particle/momentum/class_3/event/pz_residual_rms": 25.170595169067383, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04365858435630798, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20150572061538696, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19934354722499847, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.026539407670497894, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24800880253314972, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02688172087073326, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.6940900087356567, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.7608962655067444, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.415174961090088, "metrics/particle/momentum/class_4/accepted/samples": 186.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5575968027114868, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.966484546661377, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.215162754058838, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.002123832702637, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3544303774833679, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.32619285583496, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03164556995034218, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.9133776426315308, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.7386334538459778, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.247705459594727, "metrics/particle/momentum/class_4/assigned/samples": 316.0, "metrics/particle/momentum/class_4/efficiency": 0.5109890103340149, "metrics/particle/momentum/class_4/event/energy_bias": -2.748840570449829, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.163533687591553, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.929232358932495, "metrics/particle/momentum/class_4/event/px_bias": -0.5379170179367065, "metrics/particle/momentum/class_4/event/py_bias": 0.34151673316955566, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4266185760498047, "metrics/particle/momentum/class_4/event/pz_bias": 0.4796150326728821, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.112601280212402, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03137051686644554, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23501624166965485, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2392784208059311, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08727660775184631, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2789720594882965, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.051020409911870956, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.5151419639587402, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.4417831599712372, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.24968957901001, "metrics/particle/momentum/class_5/accepted/samples": 98.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.1416420340538025, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4747883975505829, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.47876042127609253, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10027017444372177, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.23188406229019165, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.0581154823303223, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09420289844274521, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.4270129203796387, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.027542417868971825, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.2454962730407715, "metrics/particle/momentum/class_5/assigned/samples": 138.0, "metrics/particle/momentum/class_5/efficiency": 0.6163522005081177, "metrics/particle/momentum/class_5/event/energy_bias": -2.1763298511505127, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.170404434204102, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.197551965713501, "metrics/particle/momentum/class_5/event/px_bias": -0.08527745306491852, "metrics/particle/momentum/class_5/event/py_bias": 0.06417213380336761, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.1983449459075928, "metrics/particle/momentum/class_5/event/pz_bias": 0.17097488045692444, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.638708591461182, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6433672308921814, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6453382968902588, "metrics/set_slot/angular_recall_all_dr0p10": 0.8423321843147278, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6385052800178528, "metrics/set_slot/soft_count_bias_mean": -0.16880647838115692}, "step_train_time": 5.192070007324219, "step_valid_time": 17286.888531684875, "step_total_time": 17292.08060193062, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.028136535878169, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2569512858963223, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2639, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2515, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8718202841096796, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8796666666666667, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.875725900116144, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12033333333333333, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8308556326395772, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8383333333333334, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8345777335324374, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8718202841096796, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002823985254519, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2427652036735375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2229, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1931, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1889, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8536693191865605, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8663077613279497, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8599421064350924, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13369223867205024, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8351016799292661, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8474652310453118, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8412380316187932, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8536693191865605, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021238811146973, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2428830865761924, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1067, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1037, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.828416149068323, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8570281124497992, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8424792735886301, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1429718875502008, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8051242236024845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8329317269076305, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8187919463087249, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.828416149068323} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_8000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..c0af91a97c5406cc3c2aef3b83756900bcacd814 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.37900322152738297, "train": {"Classification_binary": 0.35114961862564087, "Cardinality": 0.2442607879638672, "Classification": 0.23345711827278137, "Regression_pt": 0.3479152023792267, "Regression_eta": 0.004540727473795414, "Regression_sin_phi": 0.00227734143845737, "Regression_cos_phi": 0.0024352148175239563, "Regression_energy": 0.34315550327301025, "HitGrouping": 0.26517364382743835, "Auxiliary": 0.44360077381134033, "Total": 2.298335075378418, "Total_unweighted": 1.284930726280436}, "train_group": {}, "valid": {"Classification_binary": 0.3126373887062073, "Cardinality": 0.20701879262924194, "Classification": 0.20956134796142578, "Regression_pt": 0.35290201505025226, "Regression_eta": 0.004219171280662219, "Regression_sin_phi": 0.002249916239331166, "Regression_cos_phi": 0.002280355431139469, "Regression_energy": 0.3516908884048462, "HitGrouping": 0.2636125087738037, "Auxiliary": 0.4163154760996501, "Total": 2.1612566312154136, "Total_unweighted": 1.2355410830738645, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7467517852783203, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.24563133716583252, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41154244542121887, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.23559226095676422, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.21253842115402222, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6082931756973267, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7095345258712769, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.22081971168518066, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3910538852214813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2087298333644867, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.19023209810256958, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6499491930007935, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7535357475280762, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.24526962637901306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3831215798854828, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2355550229549408, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.22063137590885162, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5973039865493774, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7474060654640198, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.24319295585155487, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3929787278175354, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2331608682870865, "metrics/hit_grouping/calorimeter/all/object_purity": 0.21530042588710785, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6055623292922974, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025512516498565674, "metrics/hit_grouping/calorimeter/background_recall": 0.6257148385047913, "metrics/hit_grouping/calorimeter/completeness": 0.31576356291770935, "metrics/hit_grouping/calorimeter/energy_completeness": 0.35410580039024353, "metrics/hit_grouping/calorimeter/energy_purity": 0.3500237762928009, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5941195487976074, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7673032283782959, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.24512632191181183, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3310241997241974, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.23742011189460754, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.22374388575553894, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5613616108894348, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7313756942749023, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2601598799228668, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.44129064679145813, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.2466191053390503, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2314888834953308, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6364628672599792, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7219953536987305, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.22141951322555542, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47377070784568787, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2101207822561264, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.1805584728717804, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.666919469833374, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7187040448188782, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2651655375957489, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49018996953964233, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.25404322147369385, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2321481555700302, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6631578803062439, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7476050853729248, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3997066915035248, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6394557952880859, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3884296119213104, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3535647988319397, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.593406617641449, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7782368659973145, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.1995430290699005, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25693050026893616, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.190842404961586, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.17949527502059937, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5451988577842712, "metrics/hit_grouping/calorimeter/purity": 0.3110244870185852, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8992497324943542, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05731403827667236, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1261538416147232, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05494871363043785, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.01308811642229557, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03384615480899811, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8204828500747681, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.06418754160404205, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10490307956933975, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.06384922564029694, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.0489959642291069, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.37457239627838135, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7399664521217346, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2574504613876343, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.41523477435112, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.24669742584228516, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.22917555272579193, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6310335993766785, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7197389602661133, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4348475933074951, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.34398630261421204, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2913089394569397, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6352360844612122, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7770346999168396, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3651149868965149, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.37800103425979614, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.31147441267967224, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.49635446071624756, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6651518940925598, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4261636734008789, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3093241751194, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.27715837955474854, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6979601383209229, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6918278932571411, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42376723885536194, "metrics/hit_grouping/tracker/all/object_completeness": 0.32606372237205505, "metrics/hit_grouping/tracker/all/object_purity": 0.2845209538936615, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6611562371253967, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0007838783203624189, "metrics/hit_grouping/tracker/background_recall": 0.0023100159596651793, "metrics/hit_grouping/tracker/completeness": 0.346897155046463, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6223607063293457, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6866539716720581, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4302505552768707, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.37992313504219055, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3515099883079529, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6795444488525391, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6947199702262878, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4416523873806, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3103463351726532, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.2562177777290344, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6548638939857483, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.7002405524253845, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3903721570968628, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.22320300340652466, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.1658867448568344, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6270226240158081, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.726658284664154, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46899712085723877, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3529185950756073, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2824347913265228, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6106218695640564, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7449848055839539, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.38948068022727966, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3016357421875, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.26404544711112976, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5612516403198242, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6371060609817505, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3708733320236206, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.29537421464920044, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2908391058444977, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7445563077926636, "metrics/hit_grouping/tracker/purity": 0.32929182052612305, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9031768441200256, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.008688096888363361, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.014937933534383774, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.014733235351741314, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6878908276557922, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1285640299320221, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.09885112196207047, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.07890316098928452, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5686884522438049, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6635574102401733, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5137831568717957, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3942122757434845, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.34459349513053894, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.751136302947998, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.41583260893821716, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5247310400009155, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5725311040878296, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5585261583328247, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06989476084709167, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.02772141434252262, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.484621047973633, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.23805104196071625, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.10761120915412903, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6394340991973877, "metrics/particle/class_1/pid_accuracy": 0.9184948801994324, "metrics/particle/class_2/efficiency": 0.3008750379085541, "metrics/particle/class_2/pid_accuracy": 0.20320656895637512, "metrics/particle/class_3/efficiency": 0.4920515716075897, "metrics/particle/class_3/pid_accuracy": 0.9039500951766968, "metrics/particle/class_4/efficiency": 0.534167468547821, "metrics/particle/class_4/pid_accuracy": 0.15855856239795685, "metrics/particle/class_5/efficiency": 0.6320000290870667, "metrics/particle/class_5/pid_accuracy": 0.30063292384147644, "metrics/particle/count/bias_mean": -10.197265625, "metrics/particle/count/mae": 10.776692390441895, "metrics/particle/count/prediction_mean": 59.85872268676758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1399909406900406, "metrics/particle/event/energy_response_mean": 0.8936412334442139, "metrics/particle/event/met_abs_error": 13.446515083312988, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10946788638830185, "metrics/particle/event/scalar_pt_response_mean": 0.9260035157203674, "metrics/particle/event/vector_pt_closure": 0.09903141856193542, "metrics/particle/matched/delta_eta_abs_mean": 0.02179303765296936, "metrics/particle/matched/delta_phi_abs_mean": 0.020428968593478203, "metrics/particle/matched/delta_r_mean": 0.03326128050684929, "metrics/particle/matched/energy_relative_abs_mean": 0.2004254162311554, "metrics/particle/matched/pid_accuracy": 0.8697071671485901, "metrics/particle/matched/pt_relative_abs_mean": 0.19461016356945038, "metrics/particle/matching/duplicate_fraction": 0.098180390894413, "metrics/particle/matching/efficiency": 0.5464937090873718, "metrics/particle/matching/f1": 0.5893890857696533, "metrics/particle/matching/purity": 0.6395919322967529, "metrics/particle/matching/target_coverage_dr0p05": 0.4799174666404724, "metrics/particle/matching/target_coverage_dr0p10": 0.6275207996368408, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.33353158831596375, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6067319512367249, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03221207484602928, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19721727073192596, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19320043921470642, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.027851523831486702, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2348654866218567, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01658603921532631, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1947367191314697, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00729194562882185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.830212354660034, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5788.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4481249153614044, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7316479682922363, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.194467067718506, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.9631613492965698, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.26347723603248596, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.89762306213379, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.028821470215916634, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.721380352973938, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.131308913230896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9505155086517334, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8570.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5292611718177795, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -10.018674850463867, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 14.332927703857422, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.487005233764648, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.1632380485534668, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.1331888437271118, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.131683349609375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.09289155900478363, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.678180694580078, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.031695324927568436, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19112221896648407, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18775628507137299, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.011857587844133377, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22925972938537598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03610675036907196, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.8733644485473633, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1645074337720871, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.046608924865723, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1274.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5274770259857178, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2853906154632568, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.2734432220458984, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.0635156631469727, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26483049988746643, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 26.195247650146484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.060911018401384354, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8485074043273926, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.4121640622615814, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.721649646759033, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1888.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4217146635055542, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -17.01468849182129, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 16.76770782470703, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.4310402870178223, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09041515737771988, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.20482926070690155, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 16.4893741607666, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6810771226882935, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 35.724456787109375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03392535448074341, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20032039284706116, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19415439665317535, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.024857880547642708, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23439531028270721, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001869766740128398, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4188510775566101, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0010993382893502712, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.113531470298767, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12301.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4881759583950043, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.80116605758667, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.403170347213745, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.1872525215148926, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2998214662075043, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 22.548084259033203, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00882427953183651, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8430590033531189, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.001962279435247183, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.629254102706909, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19605.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.573820948600769, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.1071274280548096, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.367778778076172, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.071290016174316, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.9676570892333984, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.5054266452789307, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.9014973640441895, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1802726536989212, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.502297401428223, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03326648846268654, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1987876147031784, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1934482604265213, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0248973798006773, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23420169949531555, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008521406911313534, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.746479868888855, "metrics/particle/momentum/all/accepted/pz_bias": 0.0123051842674613, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0638585090637207, "metrics/particle/momentum/all/accepted/samples": 19363.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.47922682762145996, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.401087760925293, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.3983302116394043, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.1784019470214844, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.287263423204422, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 21.355159759521484, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01779596135020256, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.2193853855133057, "metrics/particle/momentum/all/assigned/pz_bias": 0.06459613144397736, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.563925266265869, "metrics/particle/momentum/all/assigned/samples": 30063.0, "metrics/particle/momentum/all/efficiency": 0.5470701456069946, "metrics/particle/momentum/all/event/energy_bias": -29.140491485595703, "metrics/particle/momentum/all/event/p3_error_mean": 35.38507080078125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.637893676757812, "metrics/particle/momentum/all/event/px_bias": 5.040480613708496, "metrics/particle/momentum/all/event/py_bias": 4.433785915374756, "metrics/particle/momentum/all/event/pz_abs_mean": 25.939123153686523, "metrics/particle/momentum/all/event/pz_bias": -0.4079130291938782, "metrics/particle/momentum/all/event/pz_residual_rms": 38.84186553955078, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037916168570518494, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2115607112646103, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.2086198478937149, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.05202709510922432, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24783921241760254, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.008912821300327778, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8490638136863708, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.029814179986715317, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.142918586730957, "metrics/particle/momentum/class_1/accepted/samples": 10771.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3045480251312256, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0896296501159668, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.2458277940750122, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9894790649414062, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.22691850364208221, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.216793060302734, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.015982037410140038, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1761329174041748, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06217627599835396, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.387392282485962, "metrics/particle/momentum/class_1/assigned/samples": 15142.0, "metrics/particle/momentum/class_1/efficiency": 0.645472526550293, "metrics/particle/momentum/class_1/event/energy_bias": 4.327407360076904, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.069242477416992, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.47120475769043, "metrics/particle/momentum/class_1/event/px_bias": 3.707322120666504, "metrics/particle/momentum/class_1/event/py_bias": 4.291553020477295, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.99567222595215, "metrics/particle/momentum/class_1/event/pz_bias": 4.538463592529297, "metrics/particle/momentum/class_1/event/pz_residual_rms": 25.995359420776367, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04594876989722252, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22294695675373077, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2190295159816742, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.058902423828840256, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25760984420776367, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8089754581451416, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.12329229712486267, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.003027439117432, "metrics/particle/momentum/class_2/accepted/samples": 868.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7122918367385864, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7175152897834778, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8636959791183472, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.45175448060035706, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3576725721359253, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.8398427963256836, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.057045064866542816, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.758694648742676, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.44646045565605164, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.589756965637207, "metrics/particle/momentum/class_2/assigned/samples": 1753.0, "metrics/particle/momentum/class_2/efficiency": 0.29156869649887085, "metrics/particle/momentum/class_2/event/energy_bias": -29.920774459838867, "metrics/particle/momentum/class_2/event/p3_error_mean": 29.534751892089844, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.653861999511719, "metrics/particle/momentum/class_2/event/px_bias": 0.15384438633918762, "metrics/particle/momentum/class_2/event/py_bias": -0.8964374661445618, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.809858322143555, "metrics/particle/momentum/class_2/event/pz_bias": 0.05015966296195984, "metrics/particle/momentum/class_2/event/pz_residual_rms": 34.798892974853516, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02480621077120304, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1768241822719574, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16777607798576355, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020215913653373718, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20866228640079498, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003500740509480238, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4244948923587799, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021846303716301918, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3793220520019531, "metrics/particle/momentum/class_3/accepted/samples": 7427.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.657286524772644, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.172586441040039, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.971759796142578, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.823582649230957, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3498741686344147, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 30.793289184570312, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013368983753025532, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0218859910964966, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019001994282007217, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.085217237472534, "metrics/particle/momentum/class_3/assigned/samples": 12716.0, "metrics/particle/momentum/class_3/efficiency": 0.48839351534843445, "metrics/particle/momentum/class_3/event/energy_bias": 2.434041976928711, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.02150535583496, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.185884475708008, "metrics/particle/momentum/class_3/event/px_bias": 2.2351627349853516, "metrics/particle/momentum/class_3/event/py_bias": 0.12918591499328613, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.021668434143066, "metrics/particle/momentum/class_3/event/pz_bias": -4.906263828277588, "metrics/particle/momentum/class_3/event/pz_residual_rms": 25.01213836669922, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04095302149653435, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.195521280169487, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18784575164318085, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.04712292179465294, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23286199569702148, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02500000037252903, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.447991967201233, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2558267116546631, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.123281240463257, "metrics/particle/momentum/class_4/accepted/samples": 200.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5208101868629456, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.117320537567139, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.648468494415283, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.4439308643341064, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.296875, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.419672966003418, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01875000074505806, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.5988502502441406, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1838667392730713, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.867614507675171, "metrics/particle/momentum/class_4/assigned/samples": 320.0, "metrics/particle/momentum/class_4/efficiency": 0.5494505763053894, "metrics/particle/momentum/class_4/event/energy_bias": -4.079489707946777, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.491507530212402, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.29213809967041, "metrics/particle/momentum/class_4/event/px_bias": -0.8295959234237671, "metrics/particle/momentum/class_4/event/py_bias": 0.648114025592804, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4650490283966064, "metrics/particle/momentum/class_4/event/pz_bias": 0.08265091478824615, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.254865646362305, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03540400043129921, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2526649832725525, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.25705626606941223, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.11645209044218063, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2986077666282654, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07216494530439377, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 3.054755210876465, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.8123077154159546, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.584705352783203, "metrics/particle/momentum/class_5/accepted/samples": 97.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.16792885959148407, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3846285045146942, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3803457021713257, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.06101352348923683, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.21969696879386902, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.522041380405426, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.12878787517547607, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.8443243503570557, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6259966492652893, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.827265739440918, "metrics/particle/momentum/class_5/assigned/samples": 132.0, "metrics/particle/momentum/class_5/efficiency": 0.6100628972053528, "metrics/particle/momentum/class_5/event/energy_bias": -1.9016740322113037, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.822901725769043, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.9110546112060547, "metrics/particle/momentum/class_5/event/px_bias": -0.22625334560871124, "metrics/particle/momentum/class_5/event/py_bias": 0.2613707482814789, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.0250532627105713, "metrics/particle/momentum/class_5/event/pz_bias": -0.17292387783527374, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.036769866943359, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6276201605796814, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6255959868431091, "metrics/set_slot/angular_recall_all_dr0p10": 0.8439027667045593, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6462836861610413, "metrics/set_slot/soft_count_bias_mean": 0.13123774528503418}, "step_train_time": 5.069538116455078, "step_valid_time": 19898.021734952927, "step_total_time": 19903.09127354622, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9971818417808567, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26750890287985385, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2945, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2634, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2524, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8701684836471755, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8943972835314091, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8821165438713998, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10560271646859083, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8338288734720846, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8570458404074703, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8452779638312123, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8701684836471755, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9345938999215271, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23970965215227702, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2171, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1934, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1901, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8549955791335102, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8908337171810226, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.872546808030679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10916628281897743, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8404067197170646, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8756333486872409, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8576584705616964, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8549955791335102, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9477632753301697, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2471258332136278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1217, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1065, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1037, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8268633540372671, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8751027115858668, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8502994011976048, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12489728841413311, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8051242236024845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8520953163516845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8279441117764471, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8268633540372671} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_9000.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a02d286a30630318535e618ec3e4ee44419e01b --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.42637862421830586, "train": {"Classification_binary": 0.34637582302093506, "Cardinality": 0.21600565314292908, "Classification": 0.2046198844909668, "Regression_pt": 0.32500195503234863, "Regression_eta": 0.004065532237291336, "Regression_sin_phi": 0.0021958383731544018, "Regression_cos_phi": 0.00214524962939322, "Regression_energy": 0.3226320445537567, "HitGrouping": 0.2550037205219269, "Auxiliary": 0.41038158535957336, "Total": 2.138742446899414, "Total_unweighted": 1.2070363273378462}, "train_group": {}, "valid": {"Classification_binary": 0.2972463369369507, "Cardinality": 0.2517783244450887, "Classification": 0.19637322425842285, "Regression_pt": 0.32119067509969074, "Regression_eta": 0.004001799660424392, "Regression_sin_phi": 0.0020625583517054715, "Regression_cos_phi": 0.002062987070530653, "Regression_energy": 0.31782110532124835, "HitGrouping": 0.2564375003178914, "Auxiliary": 0.4100704987843831, "Total": 2.09872833887736, "Total_unweighted": 1.1407586866989732, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7405518889427185, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.260953813791275, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42372721433639526, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.24959628283977509, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.23650409281253815, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6151884198188782, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6980127692222595, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.23601175844669342, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39952558279037476, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.22164088487625122, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.21604807674884796, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6607929468154907, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7464807033538818, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2585016191005707, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4064662754535675, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2484399527311325, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.24357596039772034, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.613469660282135, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7402254939079285, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.25727343559265137, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.41139817237854004, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.24640829861164093, "metrics/hit_grouping/calorimeter/all/object_purity": 0.23883113265037537, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6182674765586853, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025122035294771194, "metrics/hit_grouping/calorimeter/background_recall": 0.6234878301620483, "metrics/hit_grouping/calorimeter/completeness": 0.32940420508384705, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3727009892463684, "metrics/hit_grouping/calorimeter/energy_purity": 0.36821916699409485, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5932185649871826, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7631686925888062, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.25973057746887207, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3408181667327881, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.25120413303375244, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.24762073159217834, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5666168928146362, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7250745892524719, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2756631672382355, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.46579331159591675, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.26205870509147644, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.25633272528648376, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6518680453300476, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.707450807094574, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.23290875554084778, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.504108190536499, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.21994782984256744, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2019881308078766, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6925799250602722, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7135058641433716, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.28916603326797485, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5269386768341064, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.2774679660797119, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2608054280281067, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6743693351745605, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7260667681694031, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4184862971305847, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6588173508644104, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.40166109800338745, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3757137656211853, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6347461938858032, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.77076256275177, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.20235762000083923, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25552430748939514, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.19315055012702942, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.19769147038459778, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5558457374572754, "metrics/hit_grouping/calorimeter/purity": 0.3243304491043091, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8911465406417847, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.050908591598272324, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1584615409374237, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.048183150589466095, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012158469296991825, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04153846204280853, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8122286200523376, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.06497256457805634, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10889395326375961, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.06283604353666306, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.048656053841114044, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.38312429189682007, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7328673601150513, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.27273228764533997, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4341880977153778, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.26119160652160645, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.25460055470466614, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6440844535827637, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7250275611877441, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44590556621551514, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3719491958618164, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3198366165161133, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6153616309165955, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7841596007347107, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.39484015107154846, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.41163909435272217, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.35208162665367126, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4800897240638733, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6691570281982422, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43884381651878357, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3253501057624817, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2949278652667999, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6886666417121887, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6965013742446899, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4373701214790344, "metrics/hit_grouping/tracker/all/object_completeness": 0.34734588861465454, "metrics/hit_grouping/tracker/all/object_purity": 0.30761417746543884, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6479312181472778, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0019795214757323265, "metrics/hit_grouping/tracker/background_recall": 0.01155008003115654, "metrics/hit_grouping/tracker/completeness": 0.3706708550453186, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6098805665969849, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6985864639282227, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43517082929611206, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.40046364068984985, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3678319752216339, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6488382816314697, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.700082004070282, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46297353506088257, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3383955955505371, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.28670910000801086, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6434418559074402, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6880685091018677, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4150485396385193, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2389415204524994, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.1961505264043808, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6506876945495605, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7312862873077393, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48067936301231384, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.37687739729881287, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3107794523239136, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5930984616279602, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7175648212432861, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41944074630737305, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3282659649848938, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3213004469871521, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6238349080085754, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6474820971488953, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38410207629203796, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3122290372848511, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.30108407139778137, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7236010432243347, "metrics/hit_grouping/tracker/purity": 0.35228049755096436, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9002270698547363, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014335360378026962, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.016556011512875557, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.016030630096793175, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06776715815067291, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6789425611495972, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.13737687468528748, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.10781989246606827, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.09802370518445969, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5857957601547241, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6708178520202637, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5290124416351318, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.41956856846809387, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.37110066413879395, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7338410019874573, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4249100089073181, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4738398790359497, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.508377730846405, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5635714530944824, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.060817357152700424, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0327666811645031, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.281091690063477, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.24424076080322266, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.11380095779895782, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6365255117416382, "metrics/particle/class_1/pid_accuracy": 0.9354678392410278, "metrics/particle/class_2/efficiency": 0.29234910011291504, "metrics/particle/class_2/pid_accuracy": 0.23177282512187958, "metrics/particle/class_3/efficiency": 0.4714135527610779, "metrics/particle/class_3/pid_accuracy": 0.9057067632675171, "metrics/particle/class_4/efficiency": 0.5495668649673462, "metrics/particle/class_4/pid_accuracy": 0.22241681814193726, "metrics/particle/class_5/efficiency": 0.6819999814033508, "metrics/particle/class_5/pid_accuracy": 0.29032257199287415, "metrics/particle/count/bias_mean": -14.506510734558105, "metrics/particle/count/mae": 14.604166984558105, "metrics/particle/count/prediction_mean": 55.54948043823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1239083930850029, "metrics/particle/event/energy_response_mean": 0.9172451496124268, "metrics/particle/event/met_abs_error": 11.988322257995605, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10018811374902725, "metrics/particle/event/scalar_pt_response_mean": 0.9459525942802429, "metrics/particle/event/vector_pt_closure": 0.09338515996932983, "metrics/particle/matched/delta_eta_abs_mean": 0.02152041345834732, "metrics/particle/matched/delta_phi_abs_mean": 0.018857160583138466, "metrics/particle/matched/delta_r_mean": 0.0319669209420681, "metrics/particle/matched/energy_relative_abs_mean": 0.19085675477981567, "metrics/particle/matched/pid_accuracy": 0.8814470767974854, "metrics/particle/matched/pt_relative_abs_mean": 0.18834558129310608, "metrics/particle/matching/duplicate_fraction": 0.08354038745164871, "metrics/particle/matching/efficiency": 0.5358623266220093, "metrics/particle/matching/f1": 0.5977504849433899, "metrics/particle/matching/purity": 0.6758005023002625, "metrics/particle/matching/target_coverage_dr0p05": 0.47768712043762207, "metrics/particle/matching/target_coverage_dr0p10": 0.6198167204856873, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3346188962459564, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5855342745780945, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.030553605407476425, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18947985768318176, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1877051293849945, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011050417087972164, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22918659448623657, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014337536878883839, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.192097544670105, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04864876717329025, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.827688455581665, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5789.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32494673132896423, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4290482997894287, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4924416542053223, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3080394268035889, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.224712073802948, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.551726341247559, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025913869962096214, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.6691579818725586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06804835796356201, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.913362979888916, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 7988.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.529352605342865, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -6.121692180633545, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.436644554138184, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.667735576629639, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.7365052700042725, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.21292826533317566, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.50510025024414, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.1817190945148468, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.44304847717285, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02940749004483223, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18822774291038513, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18694858253002167, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.005424044094979763, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22747944295406342, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03659447282552719, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.014468193054199, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.38600218296051025, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.768768310546875, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1339.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3596797287464142, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0070823431015015, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9169145822525024, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7192829847335815, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2054113745689392, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.786529541015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0640530064702034, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9234249591827393, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.02042146772146225, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.064854621887207, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1811.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4432307183742523, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.340627670288086, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.276150703430176, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.132441997528076, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.12417778372764587, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08254777640104294, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.998363494873047, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.23454397916793823, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.340911865234375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.032936446368694305, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19138509035110474, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18893979489803314, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013836728408932686, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23019447922706604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001341494033113122, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.42016246914863586, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.014702590182423592, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1447118520736694, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11927.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4342517852783203, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.559187650680542, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.000640630722046, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.816349744796753, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2780751585960388, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.577455520629883, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.010541839525103569, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9027350544929504, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03234817460179329, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.2374093532562256, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 18308.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5563744902610779, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -4.892796993255615, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.324165344238281, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.00001049041748, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.6336376667022705, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.0002129077911377, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.575512409210205, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7392591238021851, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.276052474975586, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03196454793214798, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19058440625667572, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18842478096485138, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012399071827530861, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2296987622976303, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007766990456730127, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7667123079299927, "metrics/particle/momentum/all/accepted/pz_bias": -0.05110691860318184, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.2016780376434326, "metrics/particle/momentum/all/accepted/samples": 19055.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3983825445175171, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.1379966735839844, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.8508158922195435, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.6656337976455688, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.25822749733924866, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.126615524291992, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018358416855335236, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.250749945640564, "metrics/particle/momentum/all/assigned/pz_bias": 0.04172569140791893, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.920462131500244, "metrics/particle/momentum/all/assigned/samples": 28107.0, "metrics/particle/momentum/all/efficiency": 0.5383681058883667, "metrics/particle/momentum/all/event/energy_bias": -22.355117797851562, "metrics/particle/momentum/all/event/p3_error_mean": 31.65321159362793, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.757146835327148, "metrics/particle/momentum/all/event/px_bias": 3.4943206310272217, "metrics/particle/momentum/all/event/py_bias": -2.2956886291503906, "metrics/particle/momentum/all/event/pz_abs_mean": 23.308881759643555, "metrics/particle/momentum/all/event/pz_bias": 0.6864343881607056, "metrics/particle/momentum/all/event/pz_residual_rms": 35.446266174316406, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03782050311565399, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.20091432332992554, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.19875724613666534, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0034879602026194334, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.23889046907424927, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007846067659556866, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.850047767162323, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.08580397069454193, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.2632863521575928, "metrics/particle/momentum/class_1/accepted/samples": 10706.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.25360041856765747, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9637254476547241, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.040794014930725, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8514487147331238, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.22267261147499084, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.571025848388672, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.013579245656728745, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1343640089035034, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.09034809470176697, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1149301528930664, "metrics/particle/momentum/class_1/assigned/samples": 14802.0, "metrics/particle/momentum/class_1/efficiency": 0.6415773034095764, "metrics/particle/momentum/class_1/event/energy_bias": 12.60347843170166, "metrics/particle/momentum/class_1/event/p3_error_mean": 26.633834838867188, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.653120040893555, "metrics/particle/momentum/class_1/event/px_bias": 2.436964511871338, "metrics/particle/momentum/class_1/event/py_bias": 0.0976097583770752, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.220046997070312, "metrics/particle/momentum/class_1/event/pz_bias": -5.059600830078125, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.803190231323242, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04459141194820404, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2183641642332077, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2150758057832718, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06075046956539154, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.255379319190979, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.035419125109910965, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.00407075881958, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2013317048549652, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.309709548950195, "metrics/particle/momentum/class_2/accepted/samples": 847.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6019205451011658, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6722143292427063, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7048181891441345, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.30115947127342224, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2958850562572479, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.329336166381836, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.056172437965869904, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6972947120666504, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.3760239779949188, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.637895584106445, "metrics/particle/momentum/class_2/assigned/samples": 1531.0, "metrics/particle/momentum/class_2/efficiency": 0.2845146059989929, "metrics/particle/momentum/class_2/event/energy_bias": -30.291900634765625, "metrics/particle/momentum/class_2/event/p3_error_mean": 27.365257263183594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.389470100402832, "metrics/particle/momentum/class_2/event/px_bias": 1.0328683853149414, "metrics/particle/momentum/class_2/event/py_bias": -1.7617475986480713, "metrics/particle/momentum/class_2/event/pz_abs_mean": 20.49538803100586, "metrics/particle/momentum/class_2/event/pz_bias": 2.758066415786743, "metrics/particle/momentum/class_2/event/pz_residual_rms": 31.850542068481445, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02151983603835106, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17101292312145233, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16924676299095154, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03556174784898758, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21107998490333557, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0033375052735209465, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.44624683260917664, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.018760807812213898, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.5664219856262207, "metrics/particle/momentum/class_3/accepted/samples": 7191.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5641500949859619, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.866546154022217, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.04875111579895, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.900115489959717, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.30007946491241455, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.164379119873047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018716340884566307, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1751080751419067, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.1736985146999359, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.330164432525635, "metrics/particle/momentum/class_3/assigned/samples": 11327.0, "metrics/particle/momentum/class_3/efficiency": 0.47287434339523315, "metrics/particle/momentum/class_3/event/energy_bias": -1.7563869953155518, "metrics/particle/momentum/class_3/event/p3_error_mean": 18.302148818969727, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.829730987548828, "metrics/particle/momentum/class_3/event/px_bias": -0.0674150139093399, "metrics/particle/momentum/class_3/event/py_bias": -1.2125608921051025, "metrics/particle/momentum/class_3/event/pz_abs_mean": 13.186737060546875, "metrics/particle/momentum/class_3/event/pz_bias": 2.9891397953033447, "metrics/particle/momentum/class_3/event/pz_residual_rms": 23.48243522644043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03973240777850151, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20398814976215363, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1888231486082077, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028363395482301712, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23113605380058289, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02463054098188877, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.5364726781845093, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.5544213652610779, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.9247887134552, "metrics/particle/momentum/class_4/accepted/samples": 203.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3934973478317261, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.095984697341919, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.102665424346924, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9276063442230225, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2679738700389862, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.860355377197266, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.555017113685608, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.47043099999427795, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.704277276992798, "metrics/particle/momentum/class_4/assigned/samples": 306.0, "metrics/particle/momentum/class_4/efficiency": 0.557692289352417, "metrics/particle/momentum/class_4/event/energy_bias": -1.4515538215637207, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.97454571723938, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.918386936187744, "metrics/particle/momentum/class_4/event/px_bias": 0.10454535484313965, "metrics/particle/momentum/class_4/event/py_bias": -0.04275963827967644, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1930952072143555, "metrics/particle/momentum/class_4/event/pz_bias": 0.5308281779289246, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.172169208526611, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.033281292766332626, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22665853798389435, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2313462644815445, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.10281873494386673, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.27072110772132874, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.046296294778585434, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.692408800125122, "metrics/particle/momentum/class_5/accepted/pz_bias": -1.2236450910568237, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.726850986480713, "metrics/particle/momentum/class_5/accepted/samples": 108.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08130928128957748, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3880637288093567, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3784158229827881, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04466891661286354, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.19858156144618988, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7266026735305786, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07801418751478195, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.178246021270752, "metrics/particle/momentum/class_5/assigned/pz_bias": -1.2553960084915161, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.549510478973389, "metrics/particle/momentum/class_5/assigned/samples": 141.0, "metrics/particle/momentum/class_5/efficiency": 0.6792452931404114, "metrics/particle/momentum/class_5/event/energy_bias": -1.4587557315826416, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.220750331878662, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.249155282974243, "metrics/particle/momentum/class_5/event/px_bias": -0.012642428278923035, "metrics/particle/momentum/class_5/event/py_bias": 0.6237696409225464, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.192648410797119, "metrics/particle/momentum/class_5/event/pz_bias": -0.5319989323616028, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.351393699645996, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6071112155914307, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6099111437797546, "metrics/set_slot/angular_recall_all_dr0p10": 0.8498503565788269, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6448339223861694, "metrics/set_slot/soft_count_bias_mean": -3.77956223487854}, "step_train_time": 5.602181673049927, "step_valid_time": 22498.208433628082, "step_total_time": 22503.81061553955, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993934406883407, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25303727785208896, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2988, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2686, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2571, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8873472084572184, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8989290495314591, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8931005818786367, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10107095046854082, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8493557978196233, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8604417670682731, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.854862842892768, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8873472084572184, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9511649077866495, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22517069139693013, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2213, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2016, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1988, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8912466843501327, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9109805693628559, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9010055865921788, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08901943063714415, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.878868258178603, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8983280614550384, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8884916201117319, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8912466843501327, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.955338164483266, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2300492845670563, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1239, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1094, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1061, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8493788819875776, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8829701372074253, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8658488326078354, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11702986279257466, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8237577639751553, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8563357546408394, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8397309062129007, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8493788819875776} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/hyperparameters.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..e6450d47a60244c68b284125968f704c3842fa8e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 7260244, "dataset": "clic_hits", "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "set", "binary_classification_focal_gamma": 2.0, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": true, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": {"num_slots": 256, "num_layers": 4, "num_heads": 8, "ffn_multiplier": 4.0, "dropout": 0.0, "query_init": "input-conditioned", "local_attention_radius": 0.4, "attention_mode": "global-flash", "num_sectors": 32, "sector_neighbors": 1, "tracker_query_fraction": 0.6, "proposal_mode": "grid-diverse", "proposal_grid_size": 0.1, "use_aggregate_anchors": true, "aggregate_grid_size": 0.1, "presence_threshold": 0.5, "no_object_weight": 1.0, "cardinality_loss_weight": 0.05, "auxiliary_loss_weight": 0.25, "momentum_loss": "mse", "momentum_huber_delta": 0.2, "hit_grouping": true, "hit_grouping_dim": 32, "hit_grouping_chunk_size": 1024, "hit_grouping_loss_weight": 0.1, "hit_grouping_background_weight": 0.1, "hit_grouping_detector_balanced": false, "hit_grouping_tracker_particle_balanced": false, "tracker_overlap_loss_weight": 0.0, "hit_grouping_sparse": false, "hit_grouping_topk": 4, "hit_grouping_temperature": 1.0, "tracker_momentum_readout": true, "tracker_momentum_max_log_correction": 0.4054651081081644, "tracker_geometry_loss_weight": 0.0, "tracker_curvature_loss_weight": 0.0, "tracker_loss_max_hits": 64, "tracker_group_readout": false, "tracker_fit_correction": false, "tracker_fit_max_log_correction": 0.4054651081081644, "tracker_group_magnetic_field": 2.0, "event_consistency_loss_weight": 0.0, "matcher": {"presence": 1.0, "pid": 1.0, "geometry": 2.0, "pt": 1.0, "energy": 0.0, "dr_scale": 0.1, "log_pt_scale": 0.6931471805599453, "log_energy_scale": 0.6931471805599453}}, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": true, "use_flash_attn_varlen": true, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0001}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 15, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth", "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"clic_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["clic_edm_ttbar_hits", "clic_edm_ww_fullhad_hits", "clic_edm_qq_hits"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/model_kwargs.pkl b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..15d7979620643f34ff75f1190a8fdb498b085286 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af27ac504a3687a3076f06e5d6db091587b102da9f607ff8111fbb6096bc2bd2 +size 6894 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/particleflow_spec.yaml b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6f2fa8c1cdc4dcf1867b186f9de4b669cb43e580 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..06f05bcdcdd11dcf25590ee03ed55cb0d3197887 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..54e398f502818a508160fb711ec1af7b4906eb40 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..556cb152bd4aa45d285ca7ec0e85427ae6ec0cf9 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..02fc4084db54af1b8211651b57ceccc5cd8ed5d7 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..979b0d85089d0bce25d89685c99d9e4bc4695e17 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..db64450ce7b415f7b907784280d2412947a781c1 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..cdd768c25b60590c22154de33405e8a1334e0cc2 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..060e9ac82bf9cabad7f9a05bf09aa7ecc9c2bada Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..c8cd4a8bbc55e3b1f1bfdbcd888cb9e4df916426 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4d305d671bf1d362c24fa001c49c7f07e6461633 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..fee592b5b26bc481b47023b979c65a26aeaeb703 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_qq_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ea50f5060a74502ff694fcb6154acd83cb9c6496 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..07daff843eab0513e06fe321c050c3442d7cff39 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4a2887611a036b030295ddad92aea3249e87c29c Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..5215b278045f18474cde10673b5a11201229d7d2 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..104ded451a22d113adb607d6b1c309cf78d8af00 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..2648257a8b475053a527989141c780a58432edaa Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2f03190827354dc4a9e9647a62b1d1bde9fc2066 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..0dda7bfc94535648fc1824c273176313db70c2d0 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b5cf3af0263d72131e873029768b214bbf8e99ba Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..dae4b6513da63803c89333698c1ac941acc23026 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8547186e6912c1fdf5e7f57a9c610fe3d36e58ab Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..f8cb58fe66c9884d2e8a03f2839436198ac95b1e Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ttbar_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3b8278b899bc98af4678f0a075e69dc76f94d8a3 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..f42c5eac56362d63f3369b6b26482e8051f3554f Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c9b4d5c36a7c5013fcffad9fe72afd5bbe59f452 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..70c7ce81facdba478e6e687ed13fd6b9926704c0 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cafc040684814ac877d2aa803b0930e1faaf3421 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..196916197a46f0eb0dade99e2b9084d20fe03e5e Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d4c8f32763d36223db7853697c4f0656181e9aac Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..c9922d43a0b1c97b4adb4cd32332372483f347e1 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a7260f2ff1239a80b8d78b7d3d85ea4e86383e1 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..746b029b66e99e6931a918b6e8b1713d8f8a86aa Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e85bc263dd74dfcdcd4c5cf5e6d8b56b4b0d0e70 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..5ff8d3b9e179bf1495bdda58b7e1569578361e4b Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793178800.nid006678.68011.0 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793178800.nid006678.68011.0 new file mode 100644 index 0000000000000000000000000000000000000000..a52e95db0520809a2abedcd00f463f9fde7e4919 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793178800.nid006678.68011.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00bd07dae8bf81ff30f0602ec17148c586e55643c640e9945697194ab3324ba6 +size 33616079 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793278220.nid006214.91933.0 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793278220.nid006214.91933.0 new file mode 100644 index 0000000000000000000000000000000000000000..c0b9f7601f280bcbc3b1f0a286969ed7cf65f739 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/train/events.out.tfevents.1793278220.nid006214.91933.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9e03f7f9955d3d4c8b3c095eae4f49a90214fa347ce27436e195ba8ea346240 +size 14192533 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793178800.nid006678.68011.1 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793178800.nid006678.68011.1 new file mode 100644 index 0000000000000000000000000000000000000000..dc1c44cefd69d13be824caf56b0dc1b36914667f --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793178800.nid006678.68011.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b002c7c409904cfabf3998335adb693f84ef829ec524eb0e65a62ff47bcf975f +size 2205638 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793278220.nid006214.91933.1 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793278220.nid006214.91933.1 new file mode 100644 index 0000000000000000000000000000000000000000..4de27a2206ff3ed694eb353ea174218820f0bae9 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/runs/valid/events.out.tfevents.1793278220.nid006214.91933.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef8091d5387a73f97aab21dd5435748297078e6c12b0cf205b07a63bc6412f9 +size 949678 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/scenario-manifest.json b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..478b24b53a2d12f32ded5778a4c0641366ae7a29 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/scenario-manifest.json @@ -0,0 +1,1176 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239275-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239275-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "clic_hits_attention", + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "clic_hits", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "set", + "binary_classification_focal_gamma": 2.0, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": true, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": { + "num_slots": 256, + "num_layers": 4, + "num_heads": 8, + "ffn_multiplier": 4.0, + "dropout": 0.0, + "query_init": "input-conditioned", + "local_attention_radius": 0.4, + "attention_mode": "global-flash", + "num_sectors": 32, + "sector_neighbors": 1, + "tracker_query_fraction": 0.6, + "proposal_mode": "grid-diverse", + "proposal_grid_size": 0.1, + "use_aggregate_anchors": true, + "aggregate_grid_size": 0.1, + "presence_threshold": 0.5, + "no_object_weight": 1.0, + "cardinality_loss_weight": 0.05, + "auxiliary_loss_weight": 0.25, + "momentum_loss": "mse", + "momentum_huber_delta": 0.2, + "hit_grouping": true, + "hit_grouping_dim": 32, + "hit_grouping_chunk_size": 1024, + "hit_grouping_loss_weight": 0.1, + "hit_grouping_background_weight": 0.1, + "hit_grouping_detector_balanced": false, + "hit_grouping_tracker_particle_balanced": false, + "tracker_overlap_loss_weight": 0.0, + "hit_grouping_sparse": false, + "hit_grouping_topk": 4, + "hit_grouping_temperature": 1.0, + "tracker_momentum_readout": true, + "tracker_momentum_max_log_correction": 0.4054651081081644, + "tracker_geometry_loss_weight": 0.0, + "tracker_curvature_loss_weight": 0.0, + "tracker_loss_max_hits": 64, + "tracker_group_readout": false, + "tracker_fit_correction": false, + "tracker_fit_max_log_correction": 0.4054651081081644, + "tracker_group_magnetic_field": 2.0, + "event_consistency_loss_weight": 0.0, + "matcher": { + "presence": 1.0, + "pid": 1.0, + "geometry": 2.0, + "pt": 1.0, + "energy": 0.0, + "dr_scale": 0.1, + "log_pt_scale": 0.6931471805599453, + "log_energy_scale": 0.6931471805599453 + } + }, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": true, + "use_flash_attn_varlen": true, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0001 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 15, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "clic_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "clic_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "clic_edm_ttbar_hits", + "clic_edm_ww_fullhad_hits", + "clic_edm_qq_hits" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-hits-set-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "continuations": [ + { + "checkpoint": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth", + "step": 35000, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-hits-set-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--load", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "submitted_at": "2026-10-29T14:48:54.668761" + } + ] +} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train-config.yaml b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80ecb801f00c1f40096c1d63291b6f1114d85fdb --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train-config.yaml @@ -0,0 +1,329 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds +dataset: clic_hits +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- clic_edm_ttbar_hits +- clic_edm_ww_fullhad_hits +- clic_edm_qq_hits +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0001 +input_dim: 15 +load: /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: true + use_jagged_attention: true + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: 2.0 + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: true + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: set + pt_mode: direct-elemtype-split + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + dropout: 0.0 + event_consistency_loss_weight: 0.0 + ffn_multiplier: 4.0 + hit_grouping: true + hit_grouping_background_weight: 0.1 + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + hit_grouping_sparse: false + hit_grouping_temperature: 1.0 + hit_grouping_topk: 4 + hit_grouping_tracker_particle_balanced: false + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + momentum_huber_delta: 0.2 + momentum_loss: mse + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_curvature_loss_weight: 0.0 + tracker_fit_correction: false + tracker_fit_max_log_correction: 0.4054651081081644 + tracker_geometry_loss_weight: 0.0 + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_loss_max_hits: 64 + tracker_momentum_max_log_correction: 0.4054651081081644 + tracker_momentum_readout: true + tracker_overlap_loss_weight: 0.0 + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + sin_phi_mode: linear + task_queries: false + trainable: all + type: attention +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + clic_edm_qq_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 +threads: 16 +train: true +train_dataset: + clic_hits: + physical: + batch_size: 1 + samples: + clic_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +val_freq: 1000 +valid_dataset: + clic_hits: + physical: + batch_size: 1 + samples: + clic_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log new file mode 100644 index 0000000000000000000000000000000000000000..acd5780f840fdad56bb44187316212d975934c6a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log @@ -0,0 +1,20 @@ +rank0 [2026-10-28 11:12:24,689] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,885] INFO : Spawning DDP processes +rank0 [2026-10-29 14:49:18,723] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,931] INFO : Spawning DDP processes diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.0 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..a4e16fd84df71d50ad6ff986c842f3d64e4c19d6 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.0 @@ -0,0 +1,14985 @@ +rank0 [2026-10-28 11:12:32,470] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:32,471] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:57,432] INFO : Instantiating model +rank0 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:57,453] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-28 11:12:57,453] INFO : nn0 parameters: 171776 +rank0 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:57,459] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,464] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:57,464] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,468] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:57,468] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,471] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:57,471] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,474] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:57,474] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:57,477] INFO : Backbone initialization took 0.02s +rank0 [2026-10-28 11:12:57,477] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:57,477] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:57,521] INFO : Output DNNs initialization took 0.04s +rank0 [2026-10-28 11:12:57,522] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:57,522] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:57,522] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:57,522] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:57,522] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : output_mode=set +rank0 [2026-10-28 11:12:57,522] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-28 11:12:57,522] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-28 11:12:57,522] INFO : nn_pid parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : nn_pt parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : nn_eta parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : nn_energy parameters: 0 +rank0 [2026-10-28 11:12:57,523] INFO : Total MLPF parameters: 7260244 +rank0 [2026-10-28 11:12:57,523] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank0 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:57,558] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:57,560] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:57,560] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-28 11:12:57,813] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank0 [2026-10-28 11:12:57,813] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-28 11:12:57,814] INFO : Trainable parameters: 7260244 +rank0 [2026-10-28 11:12:57,814] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:57,814] INFO : Total parameters: 7260244 +rank0 [2026-10-28 11:12:57,817] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-28 11:12:57,820] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851 +rank0 [2026-10-28 11:12:57,820] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851 +rank0 [2026-10-28 11:12:57,820] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:57,820] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:02,563] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,664] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,788] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,875] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,979] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,085] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,446] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,523] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,603] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,704] INFO : train_dataset: clic_edm_ttbar_hits, 90900 +rank0 [2026-10-28 11:13:03,704] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:03,809] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:03,911] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,016] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,159] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,266] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,349] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,456] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,561] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,686] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,770] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-28 11:13:04,770] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:04,860] INFO : train_dataset: clic_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,218] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,308] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-28 11:13:05,391] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,474] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-28 11:13:05,578] INFO : train_dataset: clic_edm_qq_hits, 89996 +rank0 [2026-10-28 11:13:05,678] INFO : train_dataset: clic_edm_qq_hits, 89998 +rank0 [2026-10-28 11:13:05,787] INFO : train_dataset: clic_edm_qq_hits, 89994 +rank0 [2026-10-28 11:13:05,904] INFO : train_dataset: clic_edm_qq_hits, 89995 +rank0 [2026-10-28 11:13:06,001] INFO : train_dataset: clic_edm_qq_hits, 90099 +rank0 [2026-10-28 11:13:06,001] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:13:06,002] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:13:06,002] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:13:06,002] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,030] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,064] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,096] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,123] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,151] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,178] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,204] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,230] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,256] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,282] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,283] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,308] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,333] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,364] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,393] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,421] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,447] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,473] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,499] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,524] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,550] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,550] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,575] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,601] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,626] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,652] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,677] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,702] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,727] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,753] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,779] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,806] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,806] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:13:06,806] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:13:20,909] INFO : Got dataloaders +rank0 [2026-10-28 11:13:20,909] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:13:20,909] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:13:20,909] INFO : Starting training from step 1 to 50000 on rank 0 of 8 +rank0 [2026-10-28 11:13:20,915] INFO : Creating train_iterator +rank0 [2026-10-28 11:13:20,915] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:13:35,151] INFO : Created train_iterator +rank0 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank0 [2026-10-28 11:17:21,148] INFO : Step 100/50000 rank0 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 0.5046 | Regression_cos_phi: 0.0060 | Regression_energy: 1.2107 | Regression_eta: 0.0190 | Regression_pt: 1.1527 | Regression_sin_phi: 0.0076 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.7886s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank0 [2026-10-28 11:17:21,386] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 59.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 23% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:20:58,401] INFO : Step 200/50000 rank0 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4528 | Classification_binary: 0.5335 | Regression_cos_phi: 0.3779 | Regression_energy: 0.3743 | Regression_eta: 0.3938 | Regression_pt: 0.3975 | Regression_sin_phi: 0.4079 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 4.2693s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank0 [2026-10-28 11:20:58,639] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:24:34,342] INFO : Step 300/50000 rank0 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3465 | Classification_binary: 0.4563 | Regression_cos_phi: 0.3757 | Regression_energy: 0.3069 | Regression_eta: 0.3160 | Regression_pt: 0.3042 | Regression_sin_phi: 0.3180 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 4.4225s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank0 [2026-10-28 11:24:34,581] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:28:10,240] INFO : Step 400/50000 rank0 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 0.4375 | Regression_cos_phi: 0.3110 | Regression_energy: 0.3588 | Regression_eta: 0.2919 | Regression_pt: 0.3622 | Regression_sin_phi: 0.3620 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.2681s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank0 [2026-10-28 11:28:10,478] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:31:46,615] INFO : Step 500/50000 rank0 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2984 | Classification_binary: 0.5241 | Regression_cos_phi: 0.3371 | Regression_energy: 0.2891 | Regression_eta: 0.3190 | Regression_pt: 0.3060 | Regression_sin_phi: 0.3591 | LR: 1.00e-03 | DataLoad Time: 0.0169s | Model Forward Time: 4.7013s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank0 [2026-10-28 11:31:46,851] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:35:33,406] INFO : Step 600/50000 rank0 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2296 | Classification_binary: 0.4776 | Regression_cos_phi: 0.2510 | Regression_energy: 0.2619 | Regression_eta: 0.2740 | Regression_pt: 0.2816 | Regression_sin_phi: 0.2803 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 4.7969s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank0 [2026-10-28 11:35:33,649] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:39:10,143] INFO : Step 700/50000 rank0 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2929 | Classification_binary: 0.3763 | Regression_cos_phi: 0.3356 | Regression_energy: 0.2848 | Regression_eta: 0.2918 | Regression_pt: 0.2992 | Regression_sin_phi: 0.2614 | LR: 1.00e-03 | DataLoad Time: 0.0173s | Model Forward Time: 4.0487s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank0 [2026-10-28 11:39:10,380] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:42:49,740] INFO : Step 800/50000 rank0 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2656 | Classification_binary: 0.4947 | Regression_cos_phi: 0.2821 | Regression_energy: 0.2646 | Regression_eta: 0.3017 | Regression_pt: 0.2800 | Regression_sin_phi: 0.2703 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3312s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank0 [2026-10-28 11:42:49,977] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:46:32,597] INFO : Step 900/50000 rank0 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2391 | Classification_binary: 0.4655 | Regression_cos_phi: 0.2477 | Regression_energy: 0.3374 | Regression_eta: 0.2131 | Regression_pt: 0.3508 | Regression_sin_phi: 0.2414 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2414s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank0 [2026-10-28 11:46:32,838] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 184.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:50:10,668] INFO : Step 1000/50000 rank0 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2030 | Classification_binary: 0.4408 | Regression_cos_phi: 0.2298 | Regression_energy: 0.2497 | Regression_eta: 0.2918 | Regression_pt: 0.2616 | Regression_sin_phi: 0.3041 | LR: 9.99e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1335s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank0 [2026-10-28 11:50:10,908] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:50:10,931] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:50:11,104] INFO : Running validation on rank0 +rank0 [2026-10-28 11:50:11,104] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:50:11,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:50:59,107] INFO : VALIDATION | Step=1000/50000 | Train Loss=3.0939 | Valid Loss=3.2811 | Train Loss Unweighted=1.9421 | Valid Loss Unweighted=1.9623 | Stale=0 | ETA=1844.2m +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_success_fraction=0.375 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 11:50:59,108] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari=0.19316 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari_threshold=0.566667 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.537893 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_competing=0.596324 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.530097 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.362803 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.000707691 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 11:50:59,109] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari=0.555962 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/seconds_per_rank=22.3907 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:50:59,110] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari=0.108302 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari_threshold=0.75625 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding_minus_raw/ari=-0.0221375 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 11:50:59,111] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 11:50:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:50:59,439] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 11:50:59,456] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 11:51:19,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 11:51:20,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 11:51:21,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 11:51:22,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 11:51:32,781] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 11:51:32,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:51:33,313] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 11:51:33,317] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 11:51:49,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 11:51:50,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 11:51:51,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 11:51:52,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 11:52:02,821] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 11:52:02,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:03,470] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 11:52:03,474] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 11:52:18,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 11:52:19,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 11:52:20,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 11:52:21,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 11:52:31,289] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 11:52:31,307] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits +rank0 [2026-10-28 11:52:34,579] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits +rank0 [2026-10-28 11:52:35,904] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:37,526] INFO : Plotted jets +rank0 [2026-10-28 11:52:38,988] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:39,058] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 11:52:42,356] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 11:52:43,339] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:44,841] INFO : Plotted jets +rank0 [2026-10-28 11:52:45,979] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:46,051] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits +rank0 [2026-10-28 11:52:48,487] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits +rank0 [2026-10-28 11:52:49,325] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:50,794] INFO : Plotted jets +rank0 [2026-10-28 11:52:52,355] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:44,475] INFO : Step 1100/50000 rank0 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2165 | Classification_binary: 0.4390 | Regression_cos_phi: 0.2782 | Regression_energy: 0.3361 | Regression_eta: 0.2396 | Regression_pt: 0.3433 | Regression_sin_phi: 0.2339 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5905s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank0 [2026-10-28 11:56:44,714] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:00:31,588] INFO : Step 1200/50000 rank0 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2279 | Classification_binary: 0.5170 | Regression_cos_phi: 0.2838 | Regression_energy: 0.2895 | Regression_eta: 0.2300 | Regression_pt: 0.3017 | Regression_sin_phi: 0.3518 | LR: 9.99e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.0955s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank0 [2026-10-28 12:00:31,836] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:04:27,128] INFO : Step 1300/50000 rank0 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2066 | Classification_binary: 0.4279 | Regression_cos_phi: 0.2328 | Regression_energy: 0.2797 | Regression_eta: 0.2343 | Regression_pt: 0.2874 | Regression_sin_phi: 0.2582 | LR: 9.98e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7983s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank0 [2026-10-28 12:04:27,367] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:08:11,537] INFO : Step 1400/50000 rank0 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2061 | Classification_binary: 0.3819 | Regression_cos_phi: 0.2086 | Regression_energy: 0.2748 | Regression_eta: 0.1776 | Regression_pt: 0.2852 | Regression_sin_phi: 0.2423 | LR: 9.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4376s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank0 [2026-10-28 12:08:11,776] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 38% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 59.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:11:58,315] INFO : Step 1500/50000 rank0 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2233 | Classification_binary: 0.4867 | Regression_cos_phi: 0.2423 | Regression_energy: 0.2913 | Regression_eta: 0.1971 | Regression_pt: 0.3066 | Regression_sin_phi: 0.2158 | LR: 9.98e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.2582s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank0 [2026-10-28 12:11:58,555] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 20% +2 6 0x7408, 15961 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 98% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 92% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 9% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 5% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 88% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:15:44,378] INFO : Step 1600/50000 rank0 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2084 | Classification_binary: 0.4602 | Regression_cos_phi: 0.2402 | Regression_energy: 0.2485 | Regression_eta: 0.2307 | Regression_pt: 0.2622 | Regression_sin_phi: 0.2611 | LR: 9.98e-04 | DataLoad Time: 0.0250s | Model Forward Time: 4.8199s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank0 [2026-10-28 12:15:44,617] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:19:37,940] INFO : Step 1700/50000 rank0 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1480 | Classification_binary: 0.3332 | Regression_cos_phi: 0.1983 | Regression_energy: 0.2670 | Regression_eta: 0.2410 | Regression_pt: 0.2808 | Regression_sin_phi: 0.1987 | LR: 9.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0698s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank0 [2026-10-28 12:19:38,175] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:23:18,180] INFO : Step 1800/50000 rank0 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2317 | Classification_binary: 0.4335 | Regression_cos_phi: 0.2905 | Regression_energy: 0.2452 | Regression_eta: 0.2584 | Regression_pt: 0.2538 | Regression_sin_phi: 0.2644 | LR: 9.97e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6572s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank0 [2026-10-28 12:23:18,416] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 59.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:27:11,678] INFO : Step 1900/50000 rank0 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1973 | Classification_binary: 0.4714 | Regression_cos_phi: 0.2552 | Regression_energy: 0.2447 | Regression_eta: 0.2086 | Regression_pt: 0.2591 | Regression_sin_phi: 0.2525 | LR: 9.97e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2811s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank0 [2026-10-28 12:27:11,915] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 59.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:30:59,739] INFO : Step 2000/50000 rank0 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1844 | Classification_binary: 0.3799 | Regression_cos_phi: 0.1856 | Regression_energy: 0.2500 | Regression_eta: 0.2000 | Regression_pt: 0.2596 | Regression_sin_phi: 0.1831 | LR: 9.96e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8330s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank0 [2026-10-28 12:30:59,975] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 59.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:31:00,083] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 12:31:00,252] INFO : Running validation on rank0 +rank0 [2026-10-28 12:31:00,252] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:31:00,252] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:31:48,304] INFO : VALIDATION | Step=2000/50000 | Train Loss=2.8430 | Valid Loss=2.8879 | Train Loss Unweighted=1.6812 | Valid Loss Unweighted=1.7484 | Stale=0 | ETA=1883.0m +rank0 [2026-10-28 12:31:48,304] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:31:48,304] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_success_fraction=0.25 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 12:31:48,305] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari=0.252883 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari_threshold=0.5 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.583092 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_competing=0.651412 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.530794 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.336248 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-1.06515e-05 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 12:31:48,306] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari=0.589131 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/seconds_per_rank=21.8904 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 12:31:48,307] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari=0.103566 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari_threshold=0.7875 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding_minus_raw/ari=-0.0268738 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 12:31:48,308] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 12:31:48,311] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:48,846] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 12:31:48,857] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 12:32:13,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:32:14,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:32:16,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:32:17,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:32:27,132] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:32:27,145] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:27,679] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:32:27,683] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:32:43,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:32:44,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:32:45,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:32:47,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:32:57,108] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:32:57,119] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:57,712] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 12:32:57,715] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 12:33:13,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:33:14,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:33:15,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:33:16,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:33:26,022] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 12:33:26,173] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:33:29,454] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:33:30,536] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:32,071] INFO : Plotted jets +rank0 [2026-10-28 12:33:33,280] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:33,352] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:33:36,611] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:33:37,602] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:39,123] INFO : Plotted jets +rank0 [2026-10-28 12:33:40,637] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:40,689] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits +rank0 [2026-10-28 12:33:43,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits +rank0 [2026-10-28 12:33:43,905] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:45,434] INFO : Plotted jets +rank0 [2026-10-28 12:33:46,667] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:33,260] INFO : Step 2100/50000 rank0 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1940 | Classification_binary: 0.3425 | Regression_cos_phi: 0.2363 | Regression_energy: 0.2233 | Regression_eta: 0.2264 | Regression_pt: 0.2393 | Regression_sin_phi: 0.2330 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4264s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank0 [2026-10-28 12:37:33,498] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:41:23,702] INFO : Step 2200/50000 rank0 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2245 | Classification_binary: 0.4571 | Regression_cos_phi: 0.2661 | Regression_energy: 0.2486 | Regression_eta: 0.2621 | Regression_pt: 0.2582 | Regression_sin_phi: 0.2400 | LR: 9.96e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3804s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank0 [2026-10-28 12:41:23,943] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:45:15,171] INFO : Step 2300/50000 rank0 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2220 | Classification_binary: 0.4032 | Regression_cos_phi: 0.3147 | Regression_energy: 0.2762 | Regression_eta: 0.2594 | Regression_pt: 0.2905 | Regression_sin_phi: 0.3302 | LR: 9.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6577s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank0 [2026-10-28 12:45:15,410] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:49:09,302] INFO : Step 2400/50000 rank0 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1655 | Classification_binary: 0.3747 | Regression_cos_phi: 0.2362 | Regression_energy: 0.2378 | Regression_eta: 0.1852 | Regression_pt: 0.2527 | Regression_sin_phi: 0.1780 | LR: 9.95e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1285s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank0 [2026-10-28 12:49:09,537] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:52:58,808] INFO : Step 2500/50000 rank0 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2273 | Classification_binary: 0.4314 | Regression_cos_phi: 0.2652 | Regression_energy: 0.2444 | Regression_eta: 0.2242 | Regression_pt: 0.2593 | Regression_sin_phi: 0.2897 | LR: 9.94e-04 | DataLoad Time: 0.0136s | Model Forward Time: 5.0861s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank0 [2026-10-28 12:52:59,048] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:56:46,517] INFO : Step 2600/50000 rank0 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2004 | Classification_binary: 0.4021 | Regression_cos_phi: 0.2037 | Regression_energy: 0.2530 | Regression_eta: 0.2064 | Regression_pt: 0.2627 | Regression_sin_phi: 0.2345 | LR: 9.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9518s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank0 [2026-10-28 12:56:46,757] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 59.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:00:38,933] INFO : Step 2700/50000 rank0 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1889 | Classification_binary: 0.3339 | Regression_cos_phi: 0.2090 | Regression_energy: 0.2055 | Regression_eta: 0.1561 | Regression_pt: 0.2115 | Regression_sin_phi: 0.2020 | LR: 9.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7766s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank0 [2026-10-28 13:00:39,169] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 59.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:04:28,552] INFO : Step 2800/50000 rank0 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1707 | Classification_binary: 0.4090 | Regression_cos_phi: 0.2039 | Regression_energy: 0.2028 | Regression_eta: 0.1715 | Regression_pt: 0.2131 | Regression_sin_phi: 0.2652 | LR: 9.93e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8512s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank0 [2026-10-28 13:04:28,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:08:20,230] INFO : Step 2900/50000 rank0 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1889 | Classification_binary: 0.4170 | Regression_cos_phi: 0.2917 | Regression_energy: 0.2184 | Regression_eta: 0.2626 | Regression_pt: 0.2307 | Regression_sin_phi: 0.2809 | LR: 9.93e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9875s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank0 [2026-10-28 13:08:20,469] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 85% +4 8 0x7408, 13395 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:12:10,255] INFO : Step 3000/50000 rank0 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1673 | Classification_binary: 0.3681 | Regression_cos_phi: 0.2129 | Regression_energy: 0.2295 | Regression_eta: 0.1966 | Regression_pt: 0.2363 | Regression_sin_phi: 0.2045 | LR: 9.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1739s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank0 [2026-10-28 13:12:10,493] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 58.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:12:10,516] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 13:12:10,789] INFO : Running validation on rank0 +rank0 [2026-10-28 13:12:10,789] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:12:10,789] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:13:03,433] INFO : VALIDATION | Step=3000/50000 | Train Loss=2.7668 | Valid Loss=2.7766 | Train Loss Unweighted=1.6167 | Valid Loss Unweighted=1.6605 | Stale=0 | ETA=1875.4m +rank0 [2026-10-28 13:13:03,433] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:13:03,433] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari=0.175887 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari_threshold=0.575 +rank0 [2026-10-28 13:13:03,434] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.583429 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_competing=0.647119 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.52539 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.30984 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.00541501 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 13:13:03,435] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/seconds_per_rank=22.3829 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari=0.167186 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari_threshold=0.83125 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding_minus_raw/ari=0.0367459 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 13:13:03,436] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:13:03,442] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:03,694] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:13:03,706] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 13:13:30,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:13:31,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:13:32,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:13:33,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:13:43,489] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 13:13:43,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:44,106] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:13:44,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:00,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:14:01,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:14:02,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:14:03,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:14:13,719] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:14,292] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 13:14:14,296] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 13:14:29,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:14:30,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:14:31,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:14:31,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:14:41,713] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 13:14:42,025] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:14:45,258] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:14:46,614] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:48,189] INFO : Plotted jets +rank0 [2026-10-28 13:14:49,365] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:49,451] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:52,725] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:53,708] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:55,234] INFO : Plotted jets +rank0 [2026-10-28 13:14:56,407] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:56,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits +rank0 [2026-10-28 13:14:58,926] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits +rank0 [2026-10-28 13:15:00,185] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:01,716] INFO : Plotted jets +rank0 [2026-10-28 13:15:02,946] INFO : Plotted jet ratio +rank0 [2026-10-28 13:19:03,401] INFO : Step 3100/50000 rank0 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1644 | Classification_binary: 0.3387 | Regression_cos_phi: 0.2379 | Regression_energy: 0.2026 | Regression_eta: 0.2219 | Regression_pt: 0.2201 | Regression_sin_phi: 0.2867 | LR: 9.91e-04 | DataLoad Time: 0.0229s | Model Forward Time: 6.2398s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank0 [2026-10-28 13:19:03,639] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 57.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:23:01,349] INFO : Step 3200/50000 rank0 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1909 | Classification_binary: 0.4268 | Regression_cos_phi: 0.3201 | Regression_energy: 0.2253 | Regression_eta: 0.2215 | Regression_pt: 0.2357 | Regression_sin_phi: 0.2783 | LR: 9.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5273s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank0 [2026-10-28 13:23:01,584] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:26:54,884] INFO : Step 3300/50000 rank0 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1898 | Classification_binary: 0.3965 | Regression_cos_phi: 0.2765 | Regression_energy: 0.2031 | Regression_eta: 0.2079 | Regression_pt: 0.2136 | Regression_sin_phi: 0.2605 | LR: 9.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5027s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank0 [2026-10-28 13:26:55,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:30:49,249] INFO : Step 3400/50000 rank0 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1314 | Classification_binary: 0.2641 | Regression_cos_phi: 0.1723 | Regression_energy: 0.2548 | Regression_eta: 0.1152 | Regression_pt: 0.2661 | Regression_sin_phi: 0.1686 | LR: 9.90e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7316s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank0 [2026-10-28 13:30:49,485] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 57.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:34:48,535] INFO : Step 3500/50000 rank0 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1808 | Classification_binary: 0.4286 | Regression_cos_phi: 0.2348 | Regression_energy: 0.2189 | Regression_eta: 0.1997 | Regression_pt: 0.2331 | Regression_sin_phi: 0.2606 | LR: 9.89e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4310s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank0 [2026-10-28 13:34:48,774] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:38:46,798] INFO : Step 3600/50000 rank0 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1722 | Classification_binary: 0.4533 | Regression_cos_phi: 0.2711 | Regression_energy: 0.2208 | Regression_eta: 0.2287 | Regression_pt: 0.2392 | Regression_sin_phi: 0.2406 | LR: 9.89e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7774s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank0 [2026-10-28 13:38:47,036] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 59.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:42:36,551] INFO : Step 3700/50000 rank0 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 0.3536 | Regression_cos_phi: 0.2468 | Regression_energy: 0.2171 | Regression_eta: 0.1801 | Regression_pt: 0.2245 | Regression_sin_phi: 0.2188 | LR: 9.88e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9608s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank0 [2026-10-28 13:42:36,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:46:34,988] INFO : Step 3800/50000 rank0 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1442 | Classification_binary: 0.3506 | Regression_cos_phi: 0.1500 | Regression_energy: 0.2231 | Regression_eta: 0.1557 | Regression_pt: 0.2342 | Regression_sin_phi: 0.1763 | LR: 9.87e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6542s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank0 [2026-10-28 13:46:35,237] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:50:23,839] INFO : Step 3900/50000 rank0 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1982 | Classification_binary: 0.4473 | Regression_cos_phi: 0.2194 | Regression_energy: 0.2214 | Regression_eta: 0.1604 | Regression_pt: 0.2336 | Regression_sin_phi: 0.2326 | LR: 9.87e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9235s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank0 [2026-10-28 13:50:24,078] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:54:12,603] INFO : Step 4000/50000 rank0 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1684 | Classification_binary: 0.3552 | Regression_cos_phi: 0.1670 | Regression_energy: 0.2050 | Regression_eta: 0.1347 | Regression_pt: 0.2174 | Regression_sin_phi: 0.1856 | LR: 9.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2573s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank0 [2026-10-28 13:54:12,842] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:54:12,865] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 13:54:13,050] INFO : Running validation on rank0 +rank0 [2026-10-28 13:54:13,050] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:54:13,050] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION | Step=4000/50000 | Train Loss=2.4937 | Valid Loss=2.5816 | Train Loss Unweighted=1.5292 | Valid Loss Unweighted=1.5286 | Stale=0 | ETA=1859.7m +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:55:03,628] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari=0.197841 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari_threshold=0.575 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.588252 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_competing=0.650253 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.566429 +rank0 [2026-10-28 13:55:03,629] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.287886 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.035624 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/seconds_per_rank=22.3201 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 13:55:03,630] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari=0.175768 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari_threshold=0.83125 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding_minus_raw/ari=0.0453278 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 13:55:03,631] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:55:03,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:03,894] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:55:03,901] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 13:55:33,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:55:34,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:55:35,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:55:36,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:55:46,349] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 13:55:46,361] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:46,898] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:55:46,901] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:56:03,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:56:04,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:56:05,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:56:06,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:56:16,320] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:56:16,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:56:17,054] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 13:56:17,057] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 13:56:32,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:56:33,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:56:34,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:56:34,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:56:44,930] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 13:56:44,949] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:56:48,215] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:56:49,337] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:50,902] INFO : Plotted jets +rank0 [2026-10-28 13:56:52,070] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:52,158] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:56:55,425] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:56:56,426] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:57,947] INFO : Plotted jets +rank0 [2026-10-28 13:56:59,446] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:59,497] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits +rank0 [2026-10-28 13:57:01,781] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits +rank0 [2026-10-28 13:57:02,636] INFO : Plotted number of elements +rank0 [2026-10-28 13:57:04,168] INFO : Plotted jets +rank0 [2026-10-28 13:57:05,344] INFO : Plotted jet ratio +rank0 [2026-10-28 14:00:57,150] INFO : Step 4100/50000 rank0 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1762 | Classification_binary: 0.3449 | Regression_cos_phi: 0.1511 | Regression_energy: 0.2169 | Regression_eta: 0.1828 | Regression_pt: 0.2258 | Regression_sin_phi: 0.2037 | LR: 9.85e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7296s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank0 [2026-10-28 14:00:57,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:04:51,481] INFO : Step 4200/50000 rank0 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1874 | Classification_binary: 0.3818 | Regression_cos_phi: 0.2047 | Regression_energy: 0.2093 | Regression_eta: 0.2215 | Regression_pt: 0.2190 | Regression_sin_phi: 0.1933 | LR: 9.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.2379s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank0 [2026-10-28 14:04:51,723] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 1% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 85% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:08:42,913] INFO : Step 4300/50000 rank0 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1578 | Classification_binary: 0.3542 | Regression_cos_phi: 0.2231 | Regression_energy: 0.1764 | Regression_eta: 0.1530 | Regression_pt: 0.1822 | Regression_sin_phi: 0.1717 | LR: 9.84e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.7483s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank0 [2026-10-28 14:08:43,156] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +4 8 0x7408, 13395 58.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 95% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 0% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:12:29,838] INFO : Step 4400/50000 rank0 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1179 | Classification_binary: 0.2932 | Regression_cos_phi: 0.1903 | Regression_energy: 0.2010 | Regression_eta: 0.1474 | Regression_pt: 0.2089 | Regression_sin_phi: 0.1395 | LR: 9.83e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4208s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank0 [2026-10-28 14:12:30,075] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:16:27,576] INFO : Step 4500/50000 rank0 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1789 | Classification_binary: 0.3461 | Regression_cos_phi: 0.2262 | Regression_energy: 0.2321 | Regression_eta: 0.1894 | Regression_pt: 0.2457 | Regression_sin_phi: 0.2312 | LR: 9.82e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2152s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank0 [2026-10-28 14:16:27,814] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:20:32,517] INFO : Step 4600/50000 rank0 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1769 | Classification_binary: 0.4206 | Regression_cos_phi: 0.1808 | Regression_energy: 0.2119 | Regression_eta: 0.1590 | Regression_pt: 0.2197 | Regression_sin_phi: 0.1873 | LR: 9.81e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8261s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank0 [2026-10-28 14:20:32,756] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 59.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:24:22,859] INFO : Step 4700/50000 rank0 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.3441 | Regression_cos_phi: 0.2659 | Regression_energy: 0.1723 | Regression_eta: 0.1406 | Regression_pt: 0.1814 | Regression_sin_phi: 0.1735 | LR: 9.81e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.0330s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank0 [2026-10-28 14:24:23,099] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 5% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 59.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 68% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:28:09,931] INFO : Step 4800/50000 rank0 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.3562 | Regression_cos_phi: 0.2173 | Regression_energy: 0.2141 | Regression_eta: 0.2125 | Regression_pt: 0.2260 | Regression_sin_phi: 0.2287 | LR: 9.80e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6102s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank0 [2026-10-28 14:28:10,181] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 59.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:32:01,882] INFO : Step 4900/50000 rank0 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1652 | Classification_binary: 0.4184 | Regression_cos_phi: 0.2454 | Regression_energy: 0.2072 | Regression_eta: 0.1810 | Regression_pt: 0.2183 | Regression_sin_phi: 0.2399 | LR: 9.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.5789s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank0 [2026-10-28 14:32:02,120] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 59.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:35:53,448] INFO : Step 5000/50000 rank0 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1156 | Classification_binary: 0.3274 | Regression_cos_phi: 0.1606 | Regression_energy: 0.2046 | Regression_eta: 0.1203 | Regression_pt: 0.2136 | Regression_sin_phi: 0.1390 | LR: 9.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4919s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank0 [2026-10-28 14:35:53,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:35:53,798] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 14:35:53,983] INFO : Running validation on rank0 +rank0 [2026-10-28 14:35:53,984] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:35:53,984] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:36:42,695] INFO : VALIDATION | Step=5000/50000 | Train Loss=2.5353 | Valid Loss=2.5165 | Train Loss Unweighted=1.4509 | Valid Loss Unweighted=1.4449 | Stale=0 | ETA=1830.3m +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 14:36:42,696] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari=0.179918 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari_threshold=0.65 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.569805 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_competing=0.628194 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.542112 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.305809 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0113071 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 14:36:42,697] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/seconds_per_rank=22.3852 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari=0.187562 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari_threshold=0.80625 +rank0 [2026-10-28 14:36:42,698] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding_minus_raw/ari=0.0571222 +rank0 [2026-10-28 14:36:42,699] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 14:36:42,699] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 14:36:42,699] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 14:36:42,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:36:42,958] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 14:36:42,973] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 14:37:08,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 14:37:10,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 14:37:11,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 14:37:12,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 14:37:22,288] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 14:37:22,356] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:37:22,909] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 14:37:22,913] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:37:39,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 14:37:40,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 14:37:41,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 14:37:42,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 14:37:52,202] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 14:37:52,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:37:52,790] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 14:37:52,793] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 14:38:08,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 14:38:08,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 14:38:09,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 14:38:10,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 14:38:20,429] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 14:38:20,568] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits +rank0 [2026-10-28 14:38:24,091] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits +rank0 [2026-10-28 14:38:25,268] INFO : Plotted number of elements +rank0 [2026-10-28 14:38:26,815] INFO : Plotted jets +rank0 [2026-10-28 14:38:28,025] INFO : Plotted jet ratio +rank0 [2026-10-28 14:38:28,096] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:38:31,424] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:38:32,428] INFO : Plotted number of elements +rank0 [2026-10-28 14:38:33,951] INFO : Plotted jets +rank0 [2026-10-28 14:38:35,096] INFO : Plotted jet ratio +rank0 [2026-10-28 14:38:35,147] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits +rank0 [2026-10-28 14:38:37,820] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits +rank0 [2026-10-28 14:38:38,658] INFO : Plotted number of elements +rank0 [2026-10-28 14:38:40,184] INFO : Plotted jets +rank0 [2026-10-28 14:38:41,386] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:37,424] INFO : Step 5100/50000 rank0 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1615 | Classification_binary: 0.3075 | Regression_cos_phi: 0.1954 | Regression_energy: 0.1851 | Regression_eta: 0.1681 | Regression_pt: 0.1952 | Regression_sin_phi: 0.1910 | LR: 9.77e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1461s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank0 [2026-10-28 14:42:37,663] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:46:37,436] INFO : Step 5200/50000 rank0 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1869 | Classification_binary: 0.3924 | Regression_cos_phi: 0.1900 | Regression_energy: 0.2078 | Regression_eta: 0.1418 | Regression_pt: 0.2190 | Regression_sin_phi: 0.1739 | LR: 9.76e-04 | DataLoad Time: 0.0085s | Model Forward Time: 4.6208s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank0 [2026-10-28 14:46:37,672] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:50:34,198] INFO : Step 5300/50000 rank0 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1625 | Classification_binary: 0.3916 | Regression_cos_phi: 0.2302 | Regression_energy: 0.1995 | Regression_eta: 0.1667 | Regression_pt: 0.2087 | Regression_sin_phi: 0.2133 | LR: 9.75e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.9466s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank0 [2026-10-28 14:50:34,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 59.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:54:37,191] INFO : Step 5400/50000 rank0 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1545 | Classification_binary: 0.3144 | Regression_cos_phi: 0.1248 | Regression_energy: 0.1762 | Regression_eta: 0.1418 | Regression_pt: 0.1865 | Regression_sin_phi: 0.2231 | LR: 9.74e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1652s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank0 [2026-10-28 14:54:37,445] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:58:43,183] INFO : Step 5500/50000 rank0 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1074 | Classification_binary: 0.3045 | Regression_cos_phi: 0.1930 | Regression_energy: 0.2256 | Regression_eta: 0.1560 | Regression_pt: 0.2375 | Regression_sin_phi: 0.1939 | LR: 9.73e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3150s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank0 [2026-10-28 14:58:43,424] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:02:44,821] INFO : Step 5600/50000 rank0 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1899 | Classification_binary: 0.4245 | Regression_cos_phi: 0.2623 | Regression_energy: 0.2130 | Regression_eta: 0.2312 | Regression_pt: 0.2254 | Regression_sin_phi: 0.3587 | LR: 9.72e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3663s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank0 [2026-10-28 15:02:45,063] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 5% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 1% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 1% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:06:33,533] INFO : Step 5700/50000 rank0 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 0.3276 | Regression_cos_phi: 0.1822 | Regression_energy: 0.1862 | Regression_eta: 0.1848 | Regression_pt: 0.2023 | Regression_sin_phi: 0.1680 | LR: 9.71e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.8612s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank0 [2026-10-28 15:06:33,770] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 59.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 55% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 92% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 95% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:10:44,127] INFO : Step 5800/50000 rank0 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1490 | Classification_binary: 0.2768 | Regression_cos_phi: 0.1982 | Regression_energy: 0.2063 | Regression_eta: 0.1464 | Regression_pt: 0.2131 | Regression_sin_phi: 0.1820 | LR: 9.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1955s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank0 [2026-10-28 15:10:44,365] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:14:48,834] INFO : Step 5900/50000 rank0 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1841 | Classification_binary: 0.4298 | Regression_cos_phi: 0.2839 | Regression_energy: 0.1971 | Regression_eta: 0.2042 | Regression_pt: 0.2083 | Regression_sin_phi: 0.2583 | LR: 9.69e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.7613s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank0 [2026-10-28 15:14:49,080] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:18:46,355] INFO : Step 6000/50000 rank0 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1729 | Classification_binary: 0.3694 | Regression_cos_phi: 0.1835 | Regression_energy: 0.1941 | Regression_eta: 0.2233 | Regression_pt: 0.1993 | Regression_sin_phi: 0.2575 | LR: 9.68e-04 | DataLoad Time: 0.0104s | Model Forward Time: 7.1694s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank0 [2026-10-28 15:18:46,594] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:18:46,664] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 15:18:46,850] INFO : Running validation on rank0 +rank0 [2026-10-28 15:18:46,850] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:18:46,850] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:19:35,317] INFO : VALIDATION | Step=6000/50000 | Train Loss=2.4633 | Valid Loss=2.3247 | Train Loss Unweighted=1.4115 | Valid Loss Unweighted=1.3502 | Stale=0 | ETA=1805.8m +rank0 [2026-10-28 15:19:35,317] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:19:35,317] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 15:19:35,317] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 15:19:35,317] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari=0.357398 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari_threshold=0.675 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.527094 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_competing=0.576445 +rank0 [2026-10-28 15:19:35,318] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.535243 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.12833 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00443862 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/seconds_per_rank=22.3786 +rank0 [2026-10-28 15:19:35,319] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari=0.221068 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding_minus_raw/ari=0.0906283 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 15:19:35,320] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 15:19:35,325] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:19:35,591] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 15:19:35,595] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 15:19:56,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 15:19:57,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 15:19:59,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 15:20:01,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 15:20:11,166] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:20:11,687] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 15:20:11,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:20:27,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 15:20:28,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 15:20:29,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 15:20:30,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 15:20:40,978] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 15:20:40,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:20:41,483] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 15:20:41,486] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 15:20:57,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 15:20:58,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 15:20:58,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 15:20:59,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 15:21:09,663] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 15:21:09,679] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits +rank0 [2026-10-28 15:21:12,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits +rank0 [2026-10-28 15:21:13,967] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:15,619] INFO : Plotted jets +rank0 [2026-10-28 15:21:17,106] INFO : Plotted jet ratio +rank0 [2026-10-28 15:21:17,159] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:21:20,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:21:21,462] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:23,018] INFO : Plotted jets +rank0 [2026-10-28 15:21:24,198] INFO : Plotted jet ratio +rank0 [2026-10-28 15:21:24,261] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits +rank0 [2026-10-28 15:21:26,776] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits +rank0 [2026-10-28 15:21:27,667] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:29,211] INFO : Plotted jets +rank0 [2026-10-28 15:21:30,815] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:38,107] INFO : Step 6100/50000 rank0 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1099 | Classification_binary: 0.2802 | Regression_cos_phi: 0.1704 | Regression_energy: 0.1871 | Regression_eta: 0.1264 | Regression_pt: 0.1928 | Regression_sin_phi: 0.1687 | LR: 9.67e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1417s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank0 [2026-10-28 15:25:38,347] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:29:27,797] INFO : Step 6200/50000 rank0 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1335 | Classification_binary: 0.3003 | Regression_cos_phi: 0.1624 | Regression_energy: 0.1769 | Regression_eta: 0.1196 | Regression_pt: 0.1861 | Regression_sin_phi: 0.1363 | LR: 9.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0556s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank0 [2026-10-28 15:29:28,037] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:33:17,560] INFO : Step 6300/50000 rank0 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1517 | Classification_binary: 0.4014 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1694 | Regression_eta: 0.1687 | Regression_pt: 0.1787 | Regression_sin_phi: 0.1974 | LR: 9.65e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9680s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank0 [2026-10-28 15:33:17,800] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:37:10,147] INFO : Step 6400/50000 rank0 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1277 | Classification_binary: 0.2920 | Regression_cos_phi: 0.1927 | Regression_energy: 0.1795 | Regression_eta: 0.1412 | Regression_pt: 0.1895 | Regression_sin_phi: 0.1443 | LR: 9.64e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6515s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank0 [2026-10-28 15:37:10,397] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:41:12,311] INFO : Step 6500/50000 rank0 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1286 | Classification_binary: 0.3128 | Regression_cos_phi: 0.1432 | Regression_energy: 0.1724 | Regression_eta: 0.1574 | Regression_pt: 0.1824 | Regression_sin_phi: 0.1664 | LR: 9.63e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3522s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank0 [2026-10-28 15:41:12,562] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:45:09,711] INFO : Step 6600/50000 rank0 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1535 | Classification_binary: 0.4114 | Regression_cos_phi: 0.2321 | Regression_energy: 0.1909 | Regression_eta: 0.1689 | Regression_pt: 0.1970 | Regression_sin_phi: 0.2160 | LR: 9.62e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9217s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank0 [2026-10-28 15:45:09,947] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:49:06,753] INFO : Step 6700/50000 rank0 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.2797 | Regression_cos_phi: 0.1152 | Regression_energy: 0.1687 | Regression_eta: 0.1251 | Regression_pt: 0.1789 | Regression_sin_phi: 0.1509 | LR: 9.61e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.7160s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank0 [2026-10-28 15:49:06,993] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:52:51,484] INFO : Step 6800/50000 rank0 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1084 | Classification_binary: 0.2741 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1731 | Regression_eta: 0.1510 | Regression_pt: 0.1804 | Regression_sin_phi: 0.1599 | LR: 9.60e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5673s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank0 [2026-10-28 15:52:51,721] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:56:39,391] INFO : Step 6900/50000 rank0 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1685 | Classification_binary: 0.3482 | Regression_cos_phi: 0.2097 | Regression_energy: 0.1619 | Regression_eta: 0.1566 | Regression_pt: 0.1687 | Regression_sin_phi: 0.2094 | LR: 9.58e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5425s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank0 [2026-10-28 15:56:39,629] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:00:38,432] INFO : Step 7000/50000 rank0 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1535 | Classification_binary: 0.3800 | Regression_cos_phi: 0.2820 | Regression_energy: 0.2099 | Regression_eta: 0.1904 | Regression_pt: 0.2191 | Regression_sin_phi: 0.2165 | LR: 9.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1756s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank0 [2026-10-28 16:00:38,670] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:00:38,691] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 16:00:38,861] INFO : Running validation on rank0 +rank0 [2026-10-28 16:00:38,861] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:00:38,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION | Step=7000/50000 | Train Loss=2.5407 | Valid Loss=2.2397 | Train Loss Unweighted=1.4018 | Valid Loss Unweighted=1.2846 | Stale=0 | ETA=1770.4m +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 16:01:33,533] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari=0.373048 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.502145 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_competing=0.546247 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.539879 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.112679 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00907458 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:01:33,534] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/seconds_per_rank=22.1589 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:01:33,535] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari=0.23688 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding_minus_raw/ari=0.10644 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 16:01:33,536] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:01:33,541] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:33,794] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:01:33,798] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 16:02:00,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:02:01,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:02:02,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:02:03,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:02:13,545] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 16:02:13,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:14,068] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:02:14,072] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:02:30,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:02:31,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:02:32,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:02:33,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:02:43,598] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:02:44,276] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 16:02:44,280] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 16:03:00,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:03:01,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:03:02,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:03:03,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:03:13,101] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 16:03:13,193] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:03:16,427] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:03:17,510] INFO : Plotted number of elements +rank0 [2026-10-28 16:03:19,117] INFO : Plotted jets +rank0 [2026-10-28 16:03:20,649] INFO : Plotted jet ratio +rank0 [2026-10-28 16:03:20,702] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:03:23,868] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:03:24,838] INFO : Plotted number of elements +rank0 [2026-10-28 16:03:26,686] INFO : Plotted jets +rank0 [2026-10-28 16:03:27,835] INFO : Plotted jet ratio +rank0 [2026-10-28 16:03:27,891] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits +rank0 [2026-10-28 16:03:30,415] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits +rank0 [2026-10-28 16:03:31,247] INFO : Plotted number of elements +rank0 [2026-10-28 16:03:32,775] INFO : Plotted jets +rank0 [2026-10-28 16:03:34,332] INFO : Plotted jet ratio +rank0 [2026-10-28 16:07:40,043] INFO : Step 7100/50000 rank0 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1012 | Classification_binary: 0.2812 | Regression_cos_phi: 0.1699 | Regression_energy: 0.1693 | Regression_eta: 0.1294 | Regression_pt: 0.1749 | Regression_sin_phi: 0.1583 | LR: 9.56e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4095s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank0 [2026-10-28 16:07:40,282] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:11:41,987] INFO : Step 7200/50000 rank0 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1349 | Classification_binary: 0.3638 | Regression_cos_phi: 0.1684 | Regression_energy: 0.1598 | Regression_eta: 0.1329 | Regression_pt: 0.1659 | Regression_sin_phi: 0.1481 | LR: 9.55e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6172s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank0 [2026-10-28 16:11:42,239] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:15:46,147] INFO : Step 7300/50000 rank0 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.3358 | Regression_cos_phi: 0.1809 | Regression_energy: 0.1440 | Regression_eta: 0.1335 | Regression_pt: 0.1497 | Regression_sin_phi: 0.1792 | LR: 9.53e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6133s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank0 [2026-10-28 16:15:46,386] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:19:48,471] INFO : Step 7400/50000 rank0 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1016 | Classification_binary: 0.2966 | Regression_cos_phi: 0.1397 | Regression_energy: 0.1756 | Regression_eta: 0.1301 | Regression_pt: 0.1878 | Regression_sin_phi: 0.1314 | LR: 9.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2818s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank0 [2026-10-28 16:19:48,711] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 59.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:23:57,019] INFO : Step 7500/50000 rank0 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.2736 | Regression_cos_phi: 0.1760 | Regression_energy: 0.1488 | Regression_eta: 0.1343 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1408 | LR: 9.51e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4734s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank0 [2026-10-28 16:23:57,256] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:27:59,069] INFO : Step 7600/50000 rank0 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3637 | Regression_cos_phi: 0.1626 | Regression_energy: 0.1727 | Regression_eta: 0.1552 | Regression_pt: 0.1812 | Regression_sin_phi: 0.1630 | LR: 9.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3688s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank0 [2026-10-28 16:27:59,314] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:32:00,972] INFO : Step 7700/50000 rank0 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1058 | Classification_binary: 0.3373 | Regression_cos_phi: 0.2039 | Regression_energy: 0.1598 | Regression_eta: 0.1617 | Regression_pt: 0.1639 | Regression_sin_phi: 0.1901 | LR: 9.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6578s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank0 [2026-10-28 16:32:01,212] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:35:56,287] INFO : Step 7800/50000 rank0 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1196 | Classification_binary: 0.2616 | Regression_cos_phi: 0.1083 | Regression_energy: 0.1647 | Regression_eta: 0.1112 | Regression_pt: 0.1723 | Regression_sin_phi: 0.1703 | LR: 9.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0796s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank0 [2026-10-28 16:35:56,533] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:40:12,476] INFO : Step 7900/50000 rank0 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1595 | Classification_binary: 0.3507 | Regression_cos_phi: 0.1881 | Regression_energy: 0.1947 | Regression_eta: 0.1421 | Regression_pt: 0.2061 | Regression_sin_phi: 0.1760 | LR: 9.46e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.0147s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank0 [2026-10-28 16:40:12,717] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 59.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 52% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 8% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:44:13,675] INFO : Step 8000/50000 rank0 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1467 | Classification_binary: 0.3696 | Regression_cos_phi: 0.2367 | Regression_energy: 0.1714 | Regression_eta: 0.1719 | Regression_pt: 0.1827 | Regression_sin_phi: 0.1890 | LR: 9.44e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.0479s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank0 [2026-10-28 16:44:13,934] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:44:13,957] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 16:44:14,129] INFO : Running validation on rank0 +rank0 [2026-10-28 16:44:14,130] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:44:14,130] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:45:04,571] INFO : VALIDATION | Step=8000/50000 | Train Loss=2.2983 | Valid Loss=2.1613 | Train Loss Unweighted=1.2849 | Valid Loss Unweighted=1.2355 | Stale=0 | ETA=1741.6m +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 16:45:04,572] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari=0.415833 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.524731 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_competing=0.572531 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.558526 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0698948 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0277214 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 16:45:04,573] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/seconds_per_rank=22.4846 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari=0.238051 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding_minus_raw/ari=0.107611 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 16:45:04,574] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:45:04,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:04,832] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:45:04,836] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 16:45:30,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:45:31,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:45:33,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:45:34,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:45:44,891] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 16:45:44,904] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:45,472] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:45:45,476] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:46:01,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:46:02,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:46:03,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:46:04,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:46:14,617] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:46:14,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:46:15,126] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 16:46:15,130] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 16:46:30,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:46:31,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:46:32,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:46:32,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:46:42,905] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 16:46:43,028] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:46:46,240] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:46:47,293] INFO : Plotted number of elements +rank0 [2026-10-28 16:46:49,154] INFO : Plotted jets +rank0 [2026-10-28 16:46:50,365] INFO : Plotted jet ratio +rank0 [2026-10-28 16:46:50,435] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:46:53,637] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:46:54,671] INFO : Plotted number of elements +rank0 [2026-10-28 16:46:56,223] INFO : Plotted jets +rank0 [2026-10-28 16:46:57,392] INFO : Plotted jet ratio +rank0 [2026-10-28 16:46:57,442] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits +rank0 [2026-10-28 16:46:59,942] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits +rank0 [2026-10-28 16:47:00,808] INFO : Plotted number of elements +rank0 [2026-10-28 16:47:02,704] INFO : Plotted jets +rank0 [2026-10-28 16:47:03,904] INFO : Plotted jet ratio +rank0 [2026-10-28 16:51:08,870] INFO : Step 8100/50000 rank0 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2415 | Regression_cos_phi: 0.1022 | Regression_energy: 0.1427 | Regression_eta: 0.0780 | Regression_pt: 0.1499 | Regression_sin_phi: 0.1346 | LR: 9.43e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.7408s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank0 [2026-10-28 16:51:09,111] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:55:31,010] INFO : Step 8200/50000 rank0 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1488 | Classification_binary: 0.3467 | Regression_cos_phi: 0.1897 | Regression_energy: 0.1438 | Regression_eta: 0.1305 | Regression_pt: 0.1520 | Regression_sin_phi: 0.1728 | LR: 9.42e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7736s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank0 [2026-10-28 16:55:31,252] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 57.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:59:32,374] INFO : Step 8300/50000 rank0 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1301 | Classification_binary: 0.3542 | Regression_cos_phi: 0.1588 | Regression_energy: 0.1691 | Regression_eta: 0.2023 | Regression_pt: 0.1791 | Regression_sin_phi: 0.1806 | LR: 9.40e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.5753s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank0 [2026-10-28 16:59:32,611] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:03:27,005] INFO : Step 8400/50000 rank0 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1108 | Classification_binary: 0.2944 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1491 | Regression_eta: 0.1296 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1716 | LR: 9.39e-04 | DataLoad Time: 0.0224s | Model Forward Time: 6.7341s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank0 [2026-10-28 17:03:27,247] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 57.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:07:19,045] INFO : Step 8500/50000 rank0 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1740 | Regression_energy: 0.1470 | Regression_eta: 0.1352 | Regression_pt: 0.1556 | Regression_sin_phi: 0.1174 | LR: 9.37e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2175s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank0 [2026-10-28 17:07:19,291] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:11:31,887] INFO : Step 8600/50000 rank0 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1370 | Classification_binary: 0.3270 | Regression_cos_phi: 0.1750 | Regression_energy: 0.1749 | Regression_eta: 0.1758 | Regression_pt: 0.1798 | Regression_sin_phi: 0.2040 | LR: 9.36e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9038s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank0 [2026-10-28 17:11:32,129] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:15:45,537] INFO : Step 8700/50000 rank0 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.3606 | Regression_cos_phi: 0.1604 | Regression_energy: 0.1558 | Regression_eta: 0.1415 | Regression_pt: 0.1608 | Regression_sin_phi: 0.1931 | LR: 9.34e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.8998s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank0 [2026-10-28 17:15:45,778] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:19:41,072] INFO : Step 8800/50000 rank0 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2451 | Regression_cos_phi: 0.1376 | Regression_energy: 0.1232 | Regression_eta: 0.1121 | Regression_pt: 0.1274 | Regression_sin_phi: 0.1480 | LR: 9.33e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2644s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank0 [2026-10-28 17:19:41,312] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 0% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 78% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:23:39,094] INFO : Step 8900/50000 rank0 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.2899 | Regression_cos_phi: 0.1837 | Regression_energy: 0.1559 | Regression_eta: 0.1383 | Regression_pt: 0.1614 | Regression_sin_phi: 0.1720 | LR: 9.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3560s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank0 [2026-10-28 17:23:39,332] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:27:34,079] INFO : Step 9000/50000 rank0 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1148 | Classification_binary: 0.3528 | Regression_cos_phi: 0.1453 | Regression_energy: 0.1564 | Regression_eta: 0.1581 | Regression_pt: 0.1622 | Regression_sin_phi: 0.1549 | LR: 9.30e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.5849s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank0 [2026-10-28 17:27:34,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:27:34,354] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 17:27:34,531] INFO : Running validation on rank0 +rank0 [2026-10-28 17:27:34,531] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:27:34,531] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION | Step=9000/50000 | Train Loss=2.1387 | Valid Loss=2.0987 | Train Loss Unweighted=1.2070 | Valid Loss Unweighted=1.1408 | Stale=0 | ETA=1708.7m +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 17:28:25,259] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari=0.42491 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.47384 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_competing=0.508378 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.563571 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0608174 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0327667 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 17:28:25,260] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/seconds_per_rank=22.2811 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 17:28:25,261] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari=0.244241 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding_minus_raw/ari=0.113801 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 17:28:25,262] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 17:28:25,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:28:25,524] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 17:28:25,540] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 17:28:52,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 17:28:53,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 17:28:54,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 17:28:55,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 17:29:05,546] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:29:06,074] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 17:29:06,078] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:29:23,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 17:29:24,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 17:29:25,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 17:29:27,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 17:29:37,123] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 17:29:37,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:29:37,691] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 17:29:37,694] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 17:29:53,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 17:29:54,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 17:29:54,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 17:29:55,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 17:30:05,503] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 17:30:05,737] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits +rank0 [2026-10-28 17:30:08,985] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits +rank0 [2026-10-28 17:30:10,075] INFO : Plotted number of elements +rank0 [2026-10-28 17:30:12,065] INFO : Plotted jets +rank0 [2026-10-28 17:30:13,346] INFO : Plotted jet ratio +rank0 [2026-10-28 17:30:13,428] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:30:16,684] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:30:17,766] INFO : Plotted number of elements +rank0 [2026-10-28 17:30:19,276] INFO : Plotted jets +rank0 [2026-10-28 17:30:20,425] INFO : Plotted jet ratio +rank0 [2026-10-28 17:30:20,483] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits +rank0 [2026-10-28 17:30:22,978] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits +rank0 [2026-10-28 17:30:23,818] INFO : Plotted number of elements +rank0 [2026-10-28 17:30:25,333] INFO : Plotted jets +rank0 [2026-10-28 17:30:26,903] INFO : Plotted jet ratio +rank0 [2026-10-28 17:34:24,605] INFO : Step 9100/50000 rank0 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0893 | Classification_binary: 0.2381 | Regression_cos_phi: 0.1037 | Regression_energy: 0.1209 | Regression_eta: 0.1028 | Regression_pt: 0.1288 | Regression_sin_phi: 0.1356 | LR: 9.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8102s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank0 [2026-10-28 17:34:24,845] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:38:25,505] INFO : Step 9200/50000 rank0 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1089 | Classification_binary: 0.2785 | Regression_cos_phi: 0.1468 | Regression_energy: 0.1480 | Regression_eta: 0.1358 | Regression_pt: 0.1555 | Regression_sin_phi: 0.1609 | LR: 9.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0157s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank0 [2026-10-28 17:38:25,747] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:42:45,320] INFO : Step 9300/50000 rank0 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1164 | Classification_binary: 0.3194 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1567 | Regression_eta: 0.1364 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1402 | LR: 9.25e-04 | DataLoad Time: 0.0241s | Model Forward Time: 5.5024s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank0 [2026-10-28 17:42:45,561] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:46:51,229] INFO : Step 9400/50000 rank0 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1053 | Classification_binary: 0.2770 | Regression_cos_phi: 0.2026 | Regression_energy: 0.1629 | Regression_eta: 0.1620 | Regression_pt: 0.1691 | Regression_sin_phi: 0.1581 | LR: 9.24e-04 | DataLoad Time: 0.0107s | Model Forward Time: 5.1984s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank0 [2026-10-28 17:46:51,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 97% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 72% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:51:12,354] INFO : Step 9500/50000 rank0 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1170 | Classification_binary: 0.2309 | Regression_cos_phi: 0.1779 | Regression_energy: 0.1500 | Regression_eta: 0.1278 | Regression_pt: 0.1563 | Regression_sin_phi: 0.1679 | LR: 9.22e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9112s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank0 [2026-10-28 17:51:12,593] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:55:20,210] INFO : Step 9600/50000 rank0 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.3300 | Regression_cos_phi: 0.1836 | Regression_energy: 0.1445 | Regression_eta: 0.1332 | Regression_pt: 0.1488 | Regression_sin_phi: 0.1472 | LR: 9.21e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5738s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank0 [2026-10-28 17:55:20,450] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:59:32,568] INFO : Step 9700/50000 rank0 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.3227 | Regression_cos_phi: 0.1709 | Regression_energy: 0.1527 | Regression_eta: 0.1451 | Regression_pt: 0.1588 | Regression_sin_phi: 0.1454 | LR: 9.19e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6493s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank0 [2026-10-28 17:59:32,806] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 5% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +4 8 0x7408, 13395 58.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 94% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 98% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 89% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:03:30,184] INFO : Step 9800/50000 rank0 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2297 | Regression_cos_phi: 0.1090 | Regression_energy: 0.1213 | Regression_eta: 0.0953 | Regression_pt: 0.1225 | Regression_sin_phi: 0.1244 | LR: 9.17e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.4750s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank0 [2026-10-28 18:03:30,426] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 58.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:07:28,487] INFO : Step 9900/50000 rank0 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1031 | Classification_binary: 0.3173 | Regression_cos_phi: 0.1565 | Regression_energy: 0.1273 | Regression_eta: 0.1047 | Regression_pt: 0.1338 | Regression_sin_phi: 0.1499 | LR: 9.16e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8609s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank0 [2026-10-28 18:07:28,724] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:11:36,424] INFO : Step 10000/50000 rank0 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1177 | Classification_binary: 0.3554 | Regression_cos_phi: 0.1781 | Regression_energy: 0.1568 | Regression_eta: 0.1177 | Regression_pt: 0.1670 | Regression_sin_phi: 0.1580 | LR: 9.14e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8641s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank0 [2026-10-28 18:11:36,664] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:11:36,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 18:11:36,965] INFO : Running validation on rank0 +rank0 [2026-10-28 18:11:36,965] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:11:36,965] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:12:31,943] INFO : VALIDATION | Step=10000/50000 | Train Loss=2.0467 | Valid Loss=1.8452 | Train Loss Unweighted=1.1161 | Valid Loss Unweighted=1.0323 | Stale=0 | ETA=1676.7m +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 18:12:31,944] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari=0.432579 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.497937 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_competing=0.541266 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.573353 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0531486 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0425487 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 18:12:31,945] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/seconds_per_rank=22.4293 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari=0.255838 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding_minus_raw/ari=0.125398 +rank0 [2026-10-28 18:12:31,946] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 18:12:31,947] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 18:12:31,947] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:12:31,952] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:12:32,209] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:12:32,223] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 18:12:58,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:12:59,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:13:00,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:13:02,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:13:12,017] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:13:12,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:13:12,568] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:13:12,571] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:13:28,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:13:30,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:13:31,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:13:32,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:13:42,954] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 18:13:48,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:13:49,086] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 18:13:49,090] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 18:14:04,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:14:05,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:14:06,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:14:07,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:14:17,073] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 18:14:17,120] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:14:20,515] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:14:21,677] INFO : Plotted number of elements +rank0 [2026-10-28 18:14:23,307] INFO : Plotted jets +rank0 [2026-10-28 18:14:24,943] INFO : Plotted jet ratio +rank0 [2026-10-28 18:14:25,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:14:28,196] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:14:29,221] INFO : Plotted number of elements +rank0 [2026-10-28 18:14:30,754] INFO : Plotted jets +rank0 [2026-10-28 18:14:31,903] INFO : Plotted jet ratio +rank0 [2026-10-28 18:14:31,952] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits +rank0 [2026-10-28 18:14:34,417] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits +rank0 [2026-10-28 18:14:35,266] INFO : Plotted number of elements +rank0 [2026-10-28 18:14:36,794] INFO : Plotted jets +rank0 [2026-10-28 18:14:38,376] INFO : Plotted jet ratio +rank0 [2026-10-28 18:18:33,256] INFO : Step 10100/50000 rank0 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0958 | Classification_binary: 0.2942 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1250 | Regression_eta: 0.1080 | Regression_pt: 0.1325 | Regression_sin_phi: 0.1362 | LR: 9.12e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9179s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank0 [2026-10-28 18:18:33,499] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 35% +4 8 0x7408, 13395 57.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:22:35,188] INFO : Step 10200/50000 rank0 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.2954 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1323 | Regression_eta: 0.1189 | Regression_pt: 0.1353 | Regression_sin_phi: 0.1106 | LR: 9.11e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4587s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank0 [2026-10-28 18:22:35,432] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:26:52,970] INFO : Step 10300/50000 rank0 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1250 | Classification_binary: 0.3529 | Regression_cos_phi: 0.1815 | Regression_energy: 0.1198 | Regression_eta: 0.1358 | Regression_pt: 0.1251 | Regression_sin_phi: 0.1627 | LR: 9.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 6.0191s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank0 [2026-10-28 18:26:53,209] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:31:09,908] INFO : Step 10400/50000 rank0 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2721 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1162 | Regression_eta: 0.0832 | Regression_pt: 0.1212 | Regression_sin_phi: 0.1032 | LR: 9.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6577s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank0 [2026-10-28 18:31:10,148] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:35:26,721] INFO : Step 10500/50000 rank0 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2084 | Regression_cos_phi: 0.1079 | Regression_energy: 0.1272 | Regression_eta: 0.1325 | Regression_pt: 0.1327 | Regression_sin_phi: 0.1125 | LR: 9.06e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6028s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank0 [2026-10-28 18:35:26,961] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:39:27,628] INFO : Step 10600/50000 rank0 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1113 | Classification_binary: 0.3027 | Regression_cos_phi: 0.1262 | Regression_energy: 0.1435 | Regression_eta: 0.1382 | Regression_pt: 0.1501 | Regression_sin_phi: 0.1726 | LR: 9.04e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.9487s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank0 [2026-10-28 18:39:27,867] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:43:28,151] INFO : Step 10700/50000 rank0 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.3447 | Regression_cos_phi: 0.1428 | Regression_energy: 0.1363 | Regression_eta: 0.1081 | Regression_pt: 0.1429 | Regression_sin_phi: 0.1204 | LR: 9.02e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.9353s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank0 [2026-10-28 18:43:28,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:47:35,201] INFO : Step 10800/50000 rank0 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0796 | Regression_energy: 0.1154 | Regression_eta: 0.0873 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0890 | LR: 9.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0937s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank0 [2026-10-28 18:47:35,442] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:51:52,427] INFO : Step 10900/50000 rank0 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0904 | Classification_binary: 0.2392 | Regression_cos_phi: 0.1053 | Regression_energy: 0.1141 | Regression_eta: 0.1053 | Regression_pt: 0.1184 | Regression_sin_phi: 0.1548 | LR: 8.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2836s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank0 [2026-10-28 18:51:52,667] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:55:59,898] INFO : Step 11000/50000 rank0 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1131 | Classification_binary: 0.3355 | Regression_cos_phi: 0.1033 | Regression_energy: 0.1163 | Regression_eta: 0.1012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1169 | LR: 8.97e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5129s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank0 [2026-10-28 18:56:00,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:56:00,236] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 18:56:00,408] INFO : Running validation on rank0 +rank0 [2026-10-28 18:56:00,408] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:56:00,408] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:56:55,448] INFO : VALIDATION | Step=11000/50000 | Train Loss=1.9047 | Valid Loss=1.7062 | Train Loss Unweighted=1.0232 | Valid Loss Unweighted=0.9386 | Stale=0 | ETA=1643.6m +rank0 [2026-10-28 18:56:55,448] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:56:55,448] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari=0.436142 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 18:56:55,449] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.516697 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_competing=0.565575 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.57346 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0495855 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0426552 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 18:56:55,450] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/seconds_per_rank=22.1963 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari=0.261431 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding_minus_raw/ari=0.130991 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 18:56:55,451] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:56:55,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:56:55,714] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:56:55,720] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 18:57:22,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:57:23,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:57:24,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:57:25,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:57:35,258] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 18:57:35,271] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:57:35,837] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:57:35,841] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:57:52,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:57:53,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:57:54,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:57:55,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:58:05,718] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:58:06,769] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 18:58:06,773] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 18:58:22,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:58:23,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:58:23,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:58:24,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:58:34,834] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 18:58:35,016] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:58:38,429] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:58:39,527] INFO : Plotted number of elements +rank0 [2026-10-28 18:58:41,125] INFO : Plotted jets +rank0 [2026-10-28 18:58:42,282] INFO : Plotted jet ratio +rank0 [2026-10-28 18:58:42,337] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:58:45,526] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:58:46,547] INFO : Plotted number of elements +rank0 [2026-10-28 18:58:48,389] INFO : Plotted jets +rank0 [2026-10-28 18:58:49,560] INFO : Plotted jet ratio +rank0 [2026-10-28 18:58:49,619] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits +rank0 [2026-10-28 18:58:52,033] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits +rank0 [2026-10-28 18:58:52,891] INFO : Plotted number of elements +rank0 [2026-10-28 18:58:54,415] INFO : Plotted jets +rank0 [2026-10-28 18:58:55,621] INFO : Plotted jet ratio +rank0 [2026-10-28 19:03:04,786] INFO : Step 11100/50000 rank0 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2239 | Regression_cos_phi: 0.0872 | Regression_energy: 0.1140 | Regression_eta: 0.0691 | Regression_pt: 0.1187 | Regression_sin_phi: 0.1177 | LR: 8.95e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.1585s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank0 [2026-10-28 19:03:05,026] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 29% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 24% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 57.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 91% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:07:11,192] INFO : Step 11200/50000 rank0 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2437 | Regression_cos_phi: 0.1426 | Regression_energy: 0.1335 | Regression_eta: 0.1185 | Regression_pt: 0.1407 | Regression_sin_phi: 0.1384 | LR: 8.93e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank0 [2026-10-28 19:07:11,432] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 59.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:11:05,572] INFO : Step 11300/50000 rank0 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.3641 | Regression_cos_phi: 0.1657 | Regression_energy: 0.1402 | Regression_eta: 0.1220 | Regression_pt: 0.1468 | Regression_sin_phi: 0.1677 | LR: 8.91e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9614s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank0 [2026-10-28 19:11:05,814] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:14:57,843] INFO : Step 11400/50000 rank0 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.3400 | Regression_cos_phi: 0.1258 | Regression_energy: 0.1215 | Regression_eta: 0.1101 | Regression_pt: 0.1273 | Regression_sin_phi: 0.1394 | LR: 8.89e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7916s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank0 [2026-10-28 19:14:58,084] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 59.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 3% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:18:59,658] INFO : Step 11500/50000 rank0 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0840 | Regression_energy: 0.1410 | Regression_eta: 0.0896 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0956 | LR: 8.88e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.1675s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank0 [2026-10-28 19:18:59,899] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:23:00,066] INFO : Step 11600/50000 rank0 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1317 | Regression_energy: 0.1261 | Regression_eta: 0.0809 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0928 | LR: 8.86e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5728s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank0 [2026-10-28 19:23:00,308] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:26:58,119] INFO : Step 11700/50000 rank0 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.3110 | Regression_cos_phi: 0.1003 | Regression_energy: 0.1157 | Regression_eta: 0.1005 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1041 | LR: 8.84e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8204s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank0 [2026-10-28 19:26:58,373] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:30:54,708] INFO : Step 11800/50000 rank0 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2589 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1084 | Regression_eta: 0.0728 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0862 | LR: 8.82e-04 | DataLoad Time: 0.0231s | Model Forward Time: 6.7503s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank0 [2026-10-28 19:30:54,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 5% +2 6 0x7408, 15961 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 97% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 58.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 77% +6 10 0x7408, 62036 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:34:56,738] INFO : Step 11900/50000 rank0 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0966 | Classification_binary: 0.2648 | Regression_cos_phi: 0.0905 | Regression_energy: 0.1178 | Regression_eta: 0.1050 | Regression_pt: 0.1205 | Regression_sin_phi: 0.1273 | LR: 8.80e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.7279s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank0 [2026-10-28 19:34:56,981] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:39:04,695] INFO : Step 12000/50000 rank0 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.3076 | Regression_cos_phi: 0.1453 | Regression_energy: 0.1135 | Regression_eta: 0.1039 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1463 | LR: 8.78e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0382s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank0 [2026-10-28 19:39:04,934] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:39:04,999] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 19:39:05,169] INFO : Running validation on rank0 +rank0 [2026-10-28 19:39:05,169] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 19:39:05,169] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 19:39:55,700] INFO : VALIDATION | Step=12000/50000 | Train Loss=1.8417 | Valid Loss=1.6388 | Train Loss Unweighted=0.9564 | Valid Loss Unweighted=0.8650 | Stale=0 | ETA=1604.2m +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 19:39:55,701] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari=0.422923 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.540594 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_competing=0.596016 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.582111 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.062804 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0513059 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 19:39:55,702] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/seconds_per_rank=22.2727 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari=0.268348 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding_minus_raw/ari=0.137908 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 19:39:55,703] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 19:39:55,704] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 19:39:55,709] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:39:55,975] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 19:39:55,980] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 19:40:21,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 19:40:22,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 19:40:23,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 19:40:24,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 19:40:34,809] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 19:40:34,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:40:35,409] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 19:40:35,412] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:40:51,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 19:40:52,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 19:40:53,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 19:40:55,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 19:41:05,093] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 19:41:05,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:41:05,612] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 19:41:05,615] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 19:41:21,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 19:41:22,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 19:41:22,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 19:41:23,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 19:41:33,629] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 19:41:33,791] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits +rank0 [2026-10-28 19:41:37,054] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits +rank0 [2026-10-28 19:41:38,136] INFO : Plotted number of elements +rank0 [2026-10-28 19:41:39,644] INFO : Plotted jets +rank0 [2026-10-28 19:41:40,839] INFO : Plotted jet ratio +rank0 [2026-10-28 19:41:40,909] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:41:44,038] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:41:45,064] INFO : Plotted number of elements +rank0 [2026-10-28 19:41:47,026] INFO : Plotted jets +rank0 [2026-10-28 19:41:48,260] INFO : Plotted jet ratio +rank0 [2026-10-28 19:41:48,314] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits +rank0 [2026-10-28 19:41:50,609] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits +rank0 [2026-10-28 19:41:51,461] INFO : Plotted number of elements +rank0 [2026-10-28 19:41:52,993] INFO : Plotted jets +rank0 [2026-10-28 19:41:54,180] INFO : Plotted jet ratio +rank0 [2026-10-28 19:45:58,361] INFO : Step 12100/50000 rank0 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2860 | Regression_cos_phi: 0.1141 | Regression_energy: 0.1044 | Regression_eta: 0.0800 | Regression_pt: 0.1096 | Regression_sin_phi: 0.1006 | LR: 8.76e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.5737s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank0 [2026-10-28 19:45:58,604] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 57.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:50:03,222] INFO : Step 12200/50000 rank0 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2249 | Regression_cos_phi: 0.0930 | Regression_energy: 0.1081 | Regression_eta: 0.0806 | Regression_pt: 0.1102 | Regression_sin_phi: 0.1190 | LR: 8.74e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.7927s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank0 [2026-10-28 19:50:03,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 57.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:54:00,977] INFO : Step 12300/50000 rank0 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.3212 | Regression_cos_phi: 0.1245 | Regression_energy: 0.1192 | Regression_eta: 0.1138 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1403 | LR: 8.72e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1013s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank0 [2026-10-28 19:54:01,216] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:58:08,893] INFO : Step 12400/50000 rank0 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.3062 | Regression_cos_phi: 0.0924 | Regression_energy: 0.1069 | Regression_eta: 0.0793 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0827 | LR: 8.70e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.6650s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank0 [2026-10-28 19:58:09,132] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:02:34,871] INFO : Step 12500/50000 rank0 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2453 | Regression_cos_phi: 0.1211 | Regression_energy: 0.0951 | Regression_eta: 0.0771 | Regression_pt: 0.1001 | Regression_sin_phi: 0.1117 | LR: 8.68e-04 | DataLoad Time: 0.0233s | Model Forward Time: 5.3688s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank0 [2026-10-28 20:02:35,122] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 3% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 97% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 57.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 18% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:06:39,446] INFO : Step 12600/50000 rank0 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2687 | Regression_cos_phi: 0.1373 | Regression_energy: 0.1150 | Regression_eta: 0.0959 | Regression_pt: 0.1198 | Regression_sin_phi: 0.1083 | LR: 8.66e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.6103s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank0 [2026-10-28 20:06:39,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:10:45,927] INFO : Step 12700/50000 rank0 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.3163 | Regression_cos_phi: 0.1252 | Regression_energy: 0.1026 | Regression_eta: 0.1185 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1135 | LR: 8.64e-04 | DataLoad Time: 0.0091s | Model Forward Time: 5.9570s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank0 [2026-10-28 20:10:46,171] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:14:53,787] INFO : Step 12800/50000 rank0 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2533 | Regression_cos_phi: 0.1268 | Regression_energy: 0.1026 | Regression_eta: 0.1017 | Regression_pt: 0.1090 | Regression_sin_phi: 0.1060 | LR: 8.62e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6802s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank0 [2026-10-28 20:14:54,030] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:19:02,638] INFO : Step 12900/50000 rank0 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1140 | Regression_eta: 0.0640 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0872 | LR: 8.60e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2077s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank0 [2026-10-28 20:19:02,876] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 59.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:23:26,182] INFO : Step 13000/50000 rank0 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0997 | Classification_binary: 0.3324 | Regression_cos_phi: 0.1600 | Regression_energy: 0.1152 | Regression_eta: 0.1436 | Regression_pt: 0.1206 | Regression_sin_phi: 0.1612 | LR: 8.58e-04 | DataLoad Time: 0.0145s | Model Forward Time: 11.0158s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank0 [2026-10-28 20:23:26,422] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 6% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 17% +2 6 0x7408, 15961 49.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 56.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 50.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 90% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:23:26,530] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 20:23:26,710] INFO : Running validation on rank0 +rank0 [2026-10-28 20:23:26,710] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 20:23:26,710] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 20:24:19,228] INFO : VALIDATION | Step=13000/50000 | Train Loss=1.7741 | Valid Loss=1.5796 | Train Loss Unweighted=0.8792 | Valid Loss Unweighted=0.8251 | Stale=0 | ETA=1568.2m +rank0 [2026-10-28 20:24:19,228] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 20:24:19,229] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari=0.414775 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.547252 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_competing=0.605395 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.587971 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0709522 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0571664 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 20:24:19,230] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/seconds_per_rank=22.4002 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari=0.283617 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding_minus_raw/ari=0.153177 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 20:24:19,231] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 20:24:19,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:24:19,486] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 20:24:19,501] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 20:24:45,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 20:24:46,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 20:24:47,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 20:24:54,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 20:25:04,170] INFO : Time taken to make predictions on device 0 is: 0.35 min +rank0 [2026-10-28 20:25:04,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:25:04,756] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 20:25:04,759] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:25:20,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 20:25:21,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 20:25:23,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 20:25:24,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 20:25:34,201] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 20:25:34,217] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:25:34,724] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 20:25:34,727] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 20:25:50,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 20:25:51,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 20:25:52,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 20:25:52,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 20:26:02,938] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 20:26:03,201] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits +rank0 [2026-10-28 20:26:06,533] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits +rank0 [2026-10-28 20:26:07,604] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:09,111] INFO : Plotted jets +rank0 [2026-10-28 20:26:10,338] INFO : Plotted jet ratio +rank0 [2026-10-28 20:26:10,410] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:26:13,607] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:26:14,607] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:16,497] INFO : Plotted jets +rank0 [2026-10-28 20:26:17,692] INFO : Plotted jet ratio +rank0 [2026-10-28 20:26:17,743] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits +rank0 [2026-10-28 20:26:20,117] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits +rank0 [2026-10-28 20:26:21,015] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:22,632] INFO : Plotted jets +rank0 [2026-10-28 20:26:23,890] INFO : Plotted jet ratio +rank0 [2026-10-28 20:30:28,779] INFO : Step 13100/50000 rank0 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2515 | Regression_cos_phi: 0.0663 | Regression_energy: 0.1084 | Regression_eta: 0.0672 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0868 | LR: 8.56e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.7389s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank0 [2026-10-28 20:30:29,019] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 98% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +4 8 0x7408, 13395 57.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 55% +6 10 0x7408, 62036 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 10% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:34:39,702] INFO : Step 13200/50000 rank0 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2024 | Regression_cos_phi: 0.1177 | Regression_energy: 0.1161 | Regression_eta: 0.0702 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0757 | LR: 8.54e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.9538s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank0 [2026-10-28 20:34:39,942] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +2 6 0x7408, 15961 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 87% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 57.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 6% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 24% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 95% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:38:51,734] INFO : Step 13300/50000 rank0 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0983 | Regression_eta: 0.1009 | Regression_pt: 0.1021 | Regression_sin_phi: 0.1001 | LR: 8.52e-04 | DataLoad Time: 0.0159s | Model Forward Time: 7.2651s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank0 [2026-10-28 20:38:51,972] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 9% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 7% +2 6 0x7408, 15961 50.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 83% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 19% +4 8 0x7408, 13395 57.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 72% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 98% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:43:05,116] INFO : Step 13400/50000 rank0 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2945 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1045 | Regression_eta: 0.0725 | Regression_pt: 0.1085 | Regression_sin_phi: 0.1061 | LR: 8.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0514s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank0 [2026-10-28 20:43:05,358] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:47:14,413] INFO : Step 13500/50000 rank0 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0790 | Regression_energy: 0.1066 | Regression_eta: 0.0763 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0990 | LR: 8.48e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8839s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank0 [2026-10-28 20:47:14,654] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 57.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 23% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:51:26,902] INFO : Step 13600/50000 rank0 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0860 | Regression_eta: 0.0882 | Regression_pt: 0.0901 | Regression_sin_phi: 0.1013 | LR: 8.45e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.4582s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank0 [2026-10-28 20:51:27,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:55:42,106] INFO : Step 13700/50000 rank0 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2725 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0856 | Regression_eta: 0.0583 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0804 | LR: 8.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.5403s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank0 [2026-10-28 20:55:42,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:59:54,903] INFO : Step 13800/50000 rank0 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2500 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0877 | Regression_eta: 0.0837 | Regression_pt: 0.0914 | Regression_sin_phi: 0.1206 | LR: 8.41e-04 | DataLoad Time: 0.0228s | Model Forward Time: 6.7761s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank0 [2026-10-28 20:59:55,142] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:04:04,351] INFO : Step 13900/50000 rank0 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0880 | Regression_eta: 0.0573 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0598 | LR: 8.39e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.1015s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank0 [2026-10-28 21:04:04,605] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:08:24,743] INFO : Step 14000/50000 rank0 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.3076 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1029 | Regression_eta: 0.1227 | Regression_pt: 0.1074 | Regression_sin_phi: 0.1305 | LR: 8.37e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.5657s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank0 [2026-10-28 21:08:24,996] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:08:25,018] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 21:08:25,188] INFO : Running validation on rank0 +rank0 [2026-10-28 21:08:25,189] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:08:25,189] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:09:17,031] INFO : VALIDATION | Step=14000/50000 | Train Loss=1.6232 | Valid Loss=1.4563 | Train Loss Unweighted=0.8539 | Valid Loss Unweighted=0.7666 | Stale=0 | ETA=1532.4m +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 21:09:17,032] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari=0.422917 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.566737 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_competing=0.629767 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.607712 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0628107 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0769069 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 21:09:17,033] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/seconds_per_rank=22.0429 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:09:17,034] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari=0.298575 +rank0 [2026-10-28 21:09:17,035] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-28 21:09:17,035] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding_minus_raw/ari=0.168136 +rank0 [2026-10-28 21:09:17,035] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 21:09:17,035] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 21:09:17,035] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:09:17,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:09:17,295] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:09:17,304] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 21:09:43,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:09:44,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:09:45,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:09:46,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:09:56,413] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 21:09:56,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:09:56,996] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:09:56,999] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:10:13,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:10:14,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:10:15,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:10:16,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:10:26,311] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:10:26,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:10:26,831] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 21:10:26,835] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 21:10:42,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:10:42,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:10:43,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:10:44,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:10:54,511] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 21:10:56,429] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:10:59,792] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:11:00,848] INFO : Plotted number of elements +rank0 [2026-10-28 21:11:02,429] INFO : Plotted jets +rank0 [2026-10-28 21:11:03,572] INFO : Plotted jet ratio +rank0 [2026-10-28 21:11:03,647] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:11:06,833] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:11:07,834] INFO : Plotted number of elements +rank0 [2026-10-28 21:11:09,398] INFO : Plotted jets +rank0 [2026-10-28 21:11:10,967] INFO : Plotted jet ratio +rank0 [2026-10-28 21:11:11,018] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits +rank0 [2026-10-28 21:11:13,364] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits +rank0 [2026-10-28 21:11:14,216] INFO : Plotted number of elements +rank0 [2026-10-28 21:11:15,782] INFO : Plotted jets +rank0 [2026-10-28 21:11:16,982] INFO : Plotted jet ratio +rank0 [2026-10-28 21:15:29,539] INFO : Step 14100/50000 rank0 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2774 | Regression_cos_phi: 0.1123 | Regression_energy: 0.0829 | Regression_eta: 0.0782 | Regression_pt: 0.0860 | Regression_sin_phi: 0.1064 | LR: 8.35e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2434s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank0 [2026-10-28 21:15:29,792] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:19:49,519] INFO : Step 14200/50000 rank0 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0862 | Regression_eta: 0.0549 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0591 | LR: 8.32e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.3888s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank0 [2026-10-28 21:19:49,762] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:24:03,808] INFO : Step 14300/50000 rank0 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0837 | Regression_eta: 0.0601 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0810 | LR: 8.30e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7017s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank0 [2026-10-28 21:24:04,048] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 57.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:28:10,388] INFO : Step 14400/50000 rank0 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2844 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0851 | Regression_eta: 0.1112 | Regression_pt: 0.0917 | Regression_sin_phi: 0.1188 | LR: 8.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6857s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank0 [2026-10-28 21:28:10,627] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:32:11,792] INFO : Step 14500/50000 rank0 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0750 | Regression_eta: 0.0657 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0781 | LR: 8.26e-04 | DataLoad Time: 0.0209s | Model Forward Time: 7.0068s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank0 [2026-10-28 21:32:12,034] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 57.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:36:11,721] INFO : Step 14600/50000 rank0 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2159 | Regression_cos_phi: 0.1090 | Regression_energy: 0.0890 | Regression_eta: 0.0764 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0865 | LR: 8.24e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6455s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank0 [2026-10-28 21:36:11,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 6% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:40:17,371] INFO : Step 14700/50000 rank0 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.3186 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0895 | Regression_eta: 0.0736 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0896 | LR: 8.21e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.8994s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank0 [2026-10-28 21:40:17,613] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 92% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:44:34,580] INFO : Step 14800/50000 rank0 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2271 | Regression_cos_phi: 0.1052 | Regression_energy: 0.1085 | Regression_eta: 0.0684 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0964 | LR: 8.19e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0208s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank0 [2026-10-28 21:44:34,822] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:48:49,876] INFO : Step 14900/50000 rank0 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0963 | Regression_eta: 0.0753 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0628 | LR: 8.17e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0495s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank0 [2026-10-28 21:48:50,117] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:52:53,968] INFO : Step 15000/50000 rank0 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2155 | Regression_cos_phi: 0.0738 | Regression_energy: 0.1052 | Regression_eta: 0.0856 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0945 | LR: 8.15e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4003s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank0 [2026-10-28 21:52:54,208] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:52:54,230] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 21:52:54,406] INFO : Running validation on rank0 +rank0 [2026-10-28 21:52:54,406] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:52:54,407] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:53:44,600] INFO : VALIDATION | Step=15000/50000 | Train Loss=1.5014 | Valid Loss=1.4591 | Train Loss Unweighted=0.7740 | Valid Loss Unweighted=0.7551 | Stale=1 | ETA=1494.3m +rank0 [2026-10-28 21:53:44,600] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:53:44,600] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari=0.369356 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.519507 +rank0 [2026-10-28 21:53:44,601] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_competing=0.571192 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.611242 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.116371 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.080437 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 21:53:44,602] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/seconds_per_rank=22.2122 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari=0.283141 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding_minus_raw/ari=0.152701 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 21:53:44,603] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:53:44,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:53:44,865] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:53:44,869] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 21:54:10,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:54:11,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:54:12,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:54:14,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:54:24,109] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:54:24,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:54:24,626] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:54:24,629] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:54:41,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:54:42,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:54:43,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:54:44,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:54:54,405] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 21:54:54,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:54:54,913] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 21:54:54,916] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 21:55:10,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:55:11,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:55:12,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:55:12,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:55:22,891] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 21:55:22,992] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:55:26,320] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:55:27,429] INFO : Plotted number of elements +rank0 [2026-10-28 21:55:29,068] INFO : Plotted jets +rank0 [2026-10-28 21:55:30,248] INFO : Plotted jet ratio +rank0 [2026-10-28 21:55:30,308] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:55:33,566] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:55:34,583] INFO : Plotted number of elements +rank0 [2026-10-28 21:55:36,492] INFO : Plotted jets +rank0 [2026-10-28 21:55:37,672] INFO : Plotted jet ratio +rank0 [2026-10-28 21:55:37,724] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits +rank0 [2026-10-28 21:55:40,039] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits +rank0 [2026-10-28 21:55:40,895] INFO : Plotted number of elements +rank0 [2026-10-28 21:55:42,435] INFO : Plotted jets +rank0 [2026-10-28 21:55:43,654] INFO : Plotted jet ratio +rank0 [2026-10-28 21:59:41,476] INFO : Step 15100/50000 rank0 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0755 | Regression_eta: 0.0601 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0888 | LR: 8.12e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.3507s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank0 [2026-10-28 21:59:41,716] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 50.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 57.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:03:41,280] INFO : Step 15200/50000 rank0 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2054 | Regression_cos_phi: 0.1229 | Regression_energy: 0.0936 | Regression_eta: 0.0689 | Regression_pt: 0.0977 | Regression_sin_phi: 0.1051 | LR: 8.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.7026s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank0 [2026-10-28 22:03:41,519] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +2 6 0x7408, 15961 50.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 65% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 97% +4 8 0x7408, 13395 57.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +6 10 0x7408, 62036 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 68% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:07:36,299] INFO : Step 15300/50000 rank0 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2198 | Regression_cos_phi: 0.1166 | Regression_energy: 0.0920 | Regression_eta: 0.0786 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0880 | LR: 8.08e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5403s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank0 [2026-10-28 22:07:36,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 7% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 57.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 43% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:11:47,967] INFO : Step 15400/50000 rank0 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2604 | Regression_cos_phi: 0.0817 | Regression_energy: 0.1051 | Regression_eta: 0.0954 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1072 | LR: 8.05e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7994s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank0 [2026-10-28 22:11:48,206] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:16:07,022] INFO : Step 15500/50000 rank0 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0807 | Regression_eta: 0.1078 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0834 | LR: 8.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6173s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank0 [2026-10-28 22:16:07,263] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 57.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:20:19,025] INFO : Step 15600/50000 rank0 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0837 | Regression_eta: 0.0601 | Regression_pt: 0.0874 | Regression_sin_phi: 0.1043 | LR: 8.01e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.1189s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank0 [2026-10-28 22:20:19,264] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:24:31,095] INFO : Step 15700/50000 rank0 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.2238 | Regression_cos_phi: 0.1791 | Regression_energy: 0.0917 | Regression_eta: 0.0803 | Regression_pt: 0.0963 | Regression_sin_phi: 0.1103 | LR: 7.98e-04 | DataLoad Time: 0.0215s | Model Forward Time: 6.7593s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank0 [2026-10-28 22:24:31,349] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 57.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:28:47,099] INFO : Step 15800/50000 rank0 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2721 | Regression_cos_phi: 0.1196 | Regression_energy: 0.0889 | Regression_eta: 0.0830 | Regression_pt: 0.0933 | Regression_sin_phi: 0.1178 | LR: 7.96e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9465s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank0 [2026-10-28 22:28:47,339] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:32:58,492] INFO : Step 15900/50000 rank0 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0890 | Regression_eta: 0.0548 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0716 | LR: 7.94e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.1694s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank0 [2026-10-28 22:32:58,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 83% +2 6 0x7408, 15961 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:37:11,424] INFO : Step 16000/50000 rank0 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2295 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0665 | Regression_eta: 0.0703 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0643 | LR: 7.91e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7417s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank0 [2026-10-28 22:37:11,679] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 57.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:37:11,698] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 22:37:11,884] INFO : Running validation on rank0 +rank0 [2026-10-28 22:37:11,884] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 22:37:11,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 22:38:04,051] INFO : VALIDATION | Step=16000/50000 | Train Loss=1.4158 | Valid Loss=1.3628 | Train Loss Unweighted=0.7186 | Valid Loss Unweighted=0.7025 | Stale=0 | ETA=1455.0m +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 22:38:04,052] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari=0.41787 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.549673 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_competing=0.606377 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.622135 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0678577 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0913298 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 22:38:04,053] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/seconds_per_rank=21.9774 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari=0.276544 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-28 22:38:04,054] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding_minus_raw/ari=0.146104 +rank0 [2026-10-28 22:38:04,055] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 22:38:04,055] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 22:38:04,055] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 22:38:04,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:38:04,622] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 22:38:04,637] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 22:38:30,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 22:38:31,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 22:38:32,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 22:38:33,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 22:38:43,737] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 22:38:43,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:38:44,322] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 22:38:44,325] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:39:00,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 22:39:01,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 22:39:02,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 22:39:04,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 22:39:14,209] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 22:39:15,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:39:16,105] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 22:39:16,108] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 22:39:31,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 22:39:32,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 22:39:33,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 22:39:34,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 22:39:44,312] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 22:39:44,329] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits +rank0 [2026-10-28 22:39:47,662] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits +rank0 [2026-10-28 22:39:48,730] INFO : Plotted number of elements +rank0 [2026-10-28 22:39:50,360] INFO : Plotted jets +rank0 [2026-10-28 22:39:51,541] INFO : Plotted jet ratio +rank0 [2026-10-28 22:39:51,591] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:39:54,889] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:39:55,968] INFO : Plotted number of elements +rank0 [2026-10-28 22:39:57,528] INFO : Plotted jets +rank0 [2026-10-28 22:39:58,707] INFO : Plotted jet ratio +rank0 [2026-10-28 22:39:58,758] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits +rank0 [2026-10-28 22:40:01,520] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits +rank0 [2026-10-28 22:40:02,286] INFO : Plotted number of elements +rank0 [2026-10-28 22:40:03,820] INFO : Plotted jets +rank0 [2026-10-28 22:40:04,988] INFO : Plotted jet ratio +rank0 [2026-10-28 22:44:04,563] INFO : Step 16100/50000 rank0 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2522 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0910 | Regression_eta: 0.0641 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0790 | LR: 7.89e-04 | DataLoad Time: 0.0107s | Model Forward Time: 5.4929s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank0 [2026-10-28 22:44:04,804] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 57.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:48:09,952] INFO : Step 16200/50000 rank0 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2120 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0917 | Regression_eta: 0.0846 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0807 | LR: 7.86e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.3303s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank0 [2026-10-28 22:48:10,193] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:52:16,950] INFO : Step 16300/50000 rank0 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0933 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0870 | Regression_eta: 0.0739 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0893 | LR: 7.84e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8302s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank0 [2026-10-28 22:52:17,191] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:56:16,287] INFO : Step 16400/50000 rank0 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.2471 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0790 | Regression_eta: 0.0735 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0563 | LR: 7.81e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.0940s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank0 [2026-10-28 22:56:16,528] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 57.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:00:24,485] INFO : Step 16500/50000 rank0 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0737 | Regression_eta: 0.0695 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0815 | LR: 7.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7600s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank0 [2026-10-28 23:00:24,726] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 57.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:04:27,070] INFO : Step 16600/50000 rank0 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0790 | Regression_eta: 0.0628 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0730 | LR: 7.77e-04 | DataLoad Time: 0.0237s | Model Forward Time: 5.1141s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank0 [2026-10-28 23:04:27,310] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:08:31,857] INFO : Step 16700/50000 rank0 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2470 | Regression_cos_phi: 0.1116 | Regression_energy: 0.0852 | Regression_eta: 0.0710 | Regression_pt: 0.0884 | Regression_sin_phi: 0.1004 | LR: 7.74e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8055s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank0 [2026-10-28 23:08:32,099] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:12:29,668] INFO : Step 16800/50000 rank0 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.2254 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0978 | Regression_eta: 0.0634 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0824 | LR: 7.72e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.7578s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank0 [2026-10-28 23:12:29,910] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:16:37,590] INFO : Step 16900/50000 rank0 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0862 | Regression_eta: 0.0740 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0725 | LR: 7.69e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.4792s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank0 [2026-10-28 23:16:37,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 57.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:20:37,692] INFO : Step 17000/50000 rank0 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2418 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0826 | Regression_eta: 0.0779 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0889 | LR: 7.67e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6339s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank0 [2026-10-28 23:20:37,933] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:20:37,956] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 23:20:38,243] INFO : Running validation on rank0 +rank0 [2026-10-28 23:20:38,243] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:20:38,243] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 23:21:29,098] INFO : VALIDATION | Step=17000/50000 | Train Loss=1.3312 | Valid Loss=1.3594 | Train Loss Unweighted=0.6931 | Valid Loss Unweighted=0.6765 | Stale=0 | ETA=1413.4m +rank0 [2026-10-28 23:21:29,098] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:21:29,098] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari=0.411139 +rank0 [2026-10-28 23:21:29,099] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.553739 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_competing=0.612435 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.599737 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0745885 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0689324 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 23:21:29,100] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/seconds_per_rank=22.403 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari=0.302951 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-28 23:21:29,101] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding_minus_raw/ari=0.172511 +rank0 [2026-10-28 23:21:29,102] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 23:21:29,102] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 23:21:29,102] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 23:21:29,107] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:21:29,364] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 23:21:29,368] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 23:21:55,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 23:21:57,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 23:22:00,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 23:22:02,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 23:22:12,398] INFO : Time taken to make predictions on device 0 is: 0.32 min +rank0 [2026-10-28 23:22:12,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:22:12,960] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 23:22:12,964] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:22:29,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 23:22:30,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 23:22:31,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 23:22:32,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 23:22:42,329] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 23:22:42,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:22:42,879] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 23:22:42,883] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 23:22:59,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 23:22:59,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 23:23:01,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 23:23:02,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 23:23:12,098] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 23:23:12,117] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits +rank0 [2026-10-28 23:23:15,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits +rank0 [2026-10-28 23:23:16,587] INFO : Plotted number of elements +rank0 [2026-10-28 23:23:18,174] INFO : Plotted jets +rank0 [2026-10-28 23:23:19,313] INFO : Plotted jet ratio +rank0 [2026-10-28 23:23:19,363] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:23:22,553] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:23:23,565] INFO : Plotted number of elements +rank0 [2026-10-28 23:23:25,091] INFO : Plotted jets +rank0 [2026-10-28 23:23:26,270] INFO : Plotted jet ratio +rank0 [2026-10-28 23:23:26,334] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits +rank0 [2026-10-28 23:23:28,769] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits +rank0 [2026-10-28 23:23:29,642] INFO : Plotted number of elements +rank0 [2026-10-28 23:23:31,586] INFO : Plotted jets +rank0 [2026-10-28 23:23:32,756] INFO : Plotted jet ratio +rank0 [2026-10-28 23:27:38,987] INFO : Step 17100/50000 rank0 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.2911 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0973 | Regression_eta: 0.0907 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0983 | LR: 7.64e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.8383s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank0 [2026-10-28 23:27:39,229] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:31:39,310] INFO : Step 17200/50000 rank0 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2250 | Regression_cos_phi: 0.1159 | Regression_energy: 0.0745 | Regression_eta: 0.0739 | Regression_pt: 0.0777 | Regression_sin_phi: 0.1007 | LR: 7.62e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8336s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank0 [2026-10-28 23:31:39,552] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:35:50,429] INFO : Step 17300/50000 rank0 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2007 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0872 | Regression_eta: 0.0472 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0687 | LR: 7.59e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.3963s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank0 [2026-10-28 23:35:50,671] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +2 6 0x7408, 15961 50.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 98% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 18% +4 8 0x7408, 13395 56.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 32% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 36% +6 10 0x7408, 62036 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 96% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 67% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:39:54,491] INFO : Step 17400/50000 rank0 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2539 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0823 | Regression_eta: 0.0920 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0820 | LR: 7.57e-04 | DataLoad Time: 0.0186s | Model Forward Time: 7.1392s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank0 [2026-10-28 23:39:54,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:43:54,665] INFO : Step 17500/50000 rank0 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2155 | Regression_cos_phi: 0.1058 | Regression_energy: 0.0817 | Regression_eta: 0.0672 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0780 | LR: 7.54e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6533s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank0 [2026-10-28 23:43:54,904] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:48:09,574] INFO : Step 17600/50000 rank0 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0665 | Regression_eta: 0.0378 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0457 | LR: 7.52e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2540s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank0 [2026-10-28 23:48:09,813] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:52:18,422] INFO : Step 17700/50000 rank0 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0841 | Regression_eta: 0.0410 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0862 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5868s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank0 [2026-10-28 23:52:18,666] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:56:41,553] INFO : Step 17800/50000 rank0 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0796 | Regression_eta: 0.0822 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0985 | LR: 7.47e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.7586s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank0 [2026-10-28 23:56:41,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 57.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:00:47,846] INFO : Step 17900/50000 rank0 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0902 | Regression_eta: 0.0480 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0519 | LR: 7.44e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9881s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank0 [2026-10-29 00:00:48,090] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 3% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 88% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 93% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:04:54,385] INFO : Step 18000/50000 rank0 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2025 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0796 | Regression_eta: 0.0764 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0787 | LR: 7.42e-04 | DataLoad Time: 0.0167s | Model Forward Time: 14.2793s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank0 [2026-10-29 00:04:54,624] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 56.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 50.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:04:54,652] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-18000.pth +rank0 [2026-10-29 00:04:54,934] INFO : Running validation on rank0 +rank0 [2026-10-29 00:04:54,934] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 00:04:54,934] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION | Step=18000/50000 | Train Loss=1.3368 | Valid Loss=1.2916 | Train Loss Unweighted=0.6691 | Valid Loss Unweighted=0.6559 | Stale=0 | ETA=1373.1m +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:05:43,790] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari=0.417621 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.562514 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_competing=0.621793 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.625991 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0681061 +rank0 [2026-10-29 00:05:43,791] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.095186 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/seconds_per_rank=22.3091 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 00:05:43,792] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari=0.297893 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding_minus_raw/ari=0.167453 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 00:05:43,793] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:05:43,798] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:05:44,058] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:05:44,062] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 00:06:10,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:06:12,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:06:13,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:06:14,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:06:24,738] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 00:06:24,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:06:25,298] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:06:25,302] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:06:41,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:06:42,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:06:43,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:06:44,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:06:54,902] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:06:54,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:06:55,431] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 00:06:55,434] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 00:07:11,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:07:12,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:07:12,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:07:13,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:07:23,470] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 00:07:23,702] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:07:26,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:07:28,070] INFO : Plotted number of elements +rank0 [2026-10-29 00:07:29,644] INFO : Plotted jets +rank0 [2026-10-29 00:07:30,874] INFO : Plotted jet ratio +rank0 [2026-10-29 00:07:30,947] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:07:34,154] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:07:35,222] INFO : Plotted number of elements +rank0 [2026-10-29 00:07:36,967] INFO : Plotted jets +rank0 [2026-10-29 00:07:38,639] INFO : Plotted jet ratio +rank0 [2026-10-29 00:07:38,695] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits +rank0 [2026-10-29 00:07:41,059] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits +rank0 [2026-10-29 00:07:41,921] INFO : Plotted number of elements +rank0 [2026-10-29 00:07:43,461] INFO : Plotted jets +rank0 [2026-10-29 00:07:44,613] INFO : Plotted jet ratio +rank0 [2026-10-29 00:11:39,617] INFO : Step 18100/50000 rank0 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2149 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0714 | Regression_eta: 0.0588 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0676 | LR: 7.39e-04 | DataLoad Time: 0.0243s | Model Forward Time: 5.1370s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank0 [2026-10-29 00:11:39,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:15:48,291] INFO : Step 18200/50000 rank0 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.2033 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0800 | Regression_eta: 0.0664 | Regression_pt: 0.0832 | Regression_sin_phi: 0.1104 | LR: 7.36e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8016s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank0 [2026-10-29 00:15:48,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:19:54,958] INFO : Step 18300/50000 rank0 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0765 | Regression_eta: 0.0565 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0608 | LR: 7.34e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6752s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank0 [2026-10-29 00:19:55,199] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:24:08,075] INFO : Step 18400/50000 rank0 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2458 | Regression_cos_phi: 0.1140 | Regression_energy: 0.0720 | Regression_eta: 0.0806 | Regression_pt: 0.0749 | Regression_sin_phi: 0.1110 | LR: 7.31e-04 | DataLoad Time: 0.0142s | Model Forward Time: 5.1662s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank0 [2026-10-29 00:24:08,328] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:27:45,364] INFO : Step 18500/50000 rank0 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2362 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0822 | Regression_eta: 0.1075 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0949 | LR: 7.29e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4298s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank0 [2026-10-29 00:27:45,616] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:31:25,155] INFO : Step 18600/50000 rank0 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0958 | Regression_eta: 0.0595 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0875 | LR: 7.26e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8715s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank0 [2026-10-29 00:31:25,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:35:07,780] INFO : Step 18700/50000 rank0 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.2292 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0698 | Regression_eta: 0.0563 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0508 | LR: 7.24e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.7171s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank0 [2026-10-29 00:35:08,045] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:38:50,212] INFO : Step 18800/50000 rank0 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2656 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0906 | Regression_eta: 0.0773 | Regression_pt: 0.0932 | Regression_sin_phi: 0.1004 | LR: 7.21e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0666s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank0 [2026-10-29 00:38:50,478] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:42:28,919] INFO : Step 18900/50000 rank0 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0736 | Regression_eta: 0.0717 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0524 | LR: 7.18e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6776s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank0 [2026-10-29 00:42:29,172] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:46:10,797] INFO : Step 19000/50000 rank0 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0974 | Regression_eta: 0.0784 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0823 | LR: 7.16e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8379s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank0 [2026-10-29 00:46:11,052] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:46:11,074] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-19000.pth +rank0 [2026-10-29 00:46:11,254] INFO : Running validation on rank0 +rank0 [2026-10-29 00:46:11,254] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 00:46:11,254] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:46:59,145] INFO : VALIDATION | Step=19000/50000 | Train Loss=1.3888 | Valid Loss=1.3182 | Train Loss Unweighted=0.7071 | Valid Loss Unweighted=0.6584 | Stale=1 | ETA=1327.5m +rank0 [2026-10-29 00:46:59,145] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:46:59,145] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:46:59,145] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari=0.41952 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.577535 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_competing=0.638706 +rank0 [2026-10-29 00:46:59,146] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.599161 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0662075 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0683565 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/seconds_per_rank=21.8668 +rank0 [2026-10-29 00:46:59,147] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari=0.277293 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding_minus_raw/ari=0.146853 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 00:46:59,148] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:46:59,153] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:46:59,418] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:46:59,421] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 00:47:25,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:47:26,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:47:27,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:47:28,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:47:38,229] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:47:38,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:47:39,139] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:47:39,141] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:47:55,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:47:56,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:47:57,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:47:58,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:48:08,614] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 00:48:08,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:48:09,174] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 00:48:09,178] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 00:48:24,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:48:25,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:48:26,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:48:26,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:48:36,757] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 00:48:37,285] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:48:40,521] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:48:41,610] INFO : Plotted number of elements +rank0 [2026-10-29 00:48:43,186] INFO : Plotted jets +rank0 [2026-10-29 00:48:44,319] INFO : Plotted jet ratio +rank0 [2026-10-29 00:48:44,371] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:48:47,502] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:48:48,511] INFO : Plotted number of elements +rank0 [2026-10-29 00:48:50,033] INFO : Plotted jets +rank0 [2026-10-29 00:48:51,203] INFO : Plotted jet ratio +rank0 [2026-10-29 00:48:51,276] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits +rank0 [2026-10-29 00:48:53,733] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits +rank0 [2026-10-29 00:48:54,975] INFO : Plotted number of elements +rank0 [2026-10-29 00:48:56,505] INFO : Plotted jets +rank0 [2026-10-29 00:48:57,726] INFO : Plotted jet ratio +rank0 [2026-10-29 00:52:42,495] INFO : Step 19100/50000 rank0 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.2448 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0883 | Regression_eta: 0.0816 | Regression_pt: 0.0917 | Regression_sin_phi: 0.1177 | LR: 7.13e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.4416s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank0 [2026-10-29 00:52:42,738] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:56:32,128] INFO : Step 19200/50000 rank0 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0765 | Regression_eta: 0.0501 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0677 | LR: 7.10e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4093s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank0 [2026-10-29 00:56:32,367] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:00:16,376] INFO : Step 19300/50000 rank0 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0763 | Regression_eta: 0.0611 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0610 | LR: 7.08e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3769s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank0 [2026-10-29 01:00:16,618] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:03:58,502] INFO : Step 19400/50000 rank0 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.2127 | Regression_cos_phi: 0.1133 | Regression_energy: 0.1036 | Regression_eta: 0.0766 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0796 | LR: 7.05e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8256s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank0 [2026-10-29 01:03:58,747] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 7% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:07:42,939] INFO : Step 19500/50000 rank0 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2285 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0808 | Regression_eta: 0.0540 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0681 | LR: 7.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7837s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank0 [2026-10-29 01:07:43,179] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 59.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:11:26,590] INFO : Step 19600/50000 rank0 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1994 | Regression_cos_phi: 0.1152 | Regression_energy: 0.0823 | Regression_eta: 0.0771 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0822 | LR: 7.00e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6859s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank0 [2026-10-29 01:11:26,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:15:29,295] INFO : Step 19700/50000 rank0 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2153 | Regression_cos_phi: 0.0932 | Regression_energy: 0.1011 | Regression_eta: 0.0788 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0896 | LR: 6.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.1424s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank0 [2026-10-29 01:15:29,539] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 70% +2 6 0x7408, 15961 50.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 98% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 57.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 1% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 98% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:19:25,447] INFO : Step 19800/50000 rank0 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0676 | Regression_eta: 0.0741 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0831 | LR: 6.94e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9888s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank0 [2026-10-29 01:19:25,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:23:27,020] INFO : Step 19900/50000 rank0 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2234 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0749 | Regression_eta: 0.0646 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0665 | LR: 6.92e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.4520s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank0 [2026-10-29 01:23:27,263] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 57.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:27:16,605] INFO : Step 20000/50000 rank0 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2275 | Regression_cos_phi: 0.1052 | Regression_energy: 0.0724 | Regression_eta: 0.0774 | Regression_pt: 0.0754 | Regression_sin_phi: 0.1075 | LR: 6.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7069s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank0 [2026-10-29 01:27:16,844] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 7% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 87% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:27:16,869] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-20000.pth +rank0 [2026-10-29 01:27:17,126] INFO : Running validation on rank0 +rank0 [2026-10-29 01:27:17,127] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 01:27:17,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 01:28:07,136] INFO : VALIDATION | Step=20000/50000 | Train Loss=1.3131 | Valid Loss=1.3310 | Train Loss Unweighted=0.6575 | Valid Loss Unweighted=0.6733 | Stale=2 | ETA=1282.2m +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 01:28:07,137] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari=0.466147 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.525572 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_competing=0.575959 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.597607 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0195807 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0668024 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 01:28:07,138] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/seconds_per_rank=21.9651 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 01:28:07,139] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari=0.266587 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding_minus_raw/ari=0.136147 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 01:28:07,140] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 01:28:07,145] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:28:07,399] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 01:28:07,408] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 01:28:34,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 01:28:35,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 01:28:36,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 01:28:37,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 01:28:47,957] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 01:28:48,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:28:48,586] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 01:28:48,590] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:29:04,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 01:29:05,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 01:29:06,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 01:29:07,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 01:29:18,005] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 01:29:18,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:29:18,549] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 01:29:18,552] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 01:29:33,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 01:29:34,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 01:29:35,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 01:29:35,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 01:29:45,982] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 01:29:46,393] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:29:49,618] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:29:50,680] INFO : Plotted number of elements +rank0 [2026-10-29 01:29:52,622] INFO : Plotted jets +rank0 [2026-10-29 01:29:53,827] INFO : Plotted jet ratio +rank0 [2026-10-29 01:29:53,878] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:29:57,092] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:29:58,110] INFO : Plotted number of elements +rank0 [2026-10-29 01:29:59,651] INFO : Plotted jets +rank0 [2026-10-29 01:30:00,864] INFO : Plotted jet ratio +rank0 [2026-10-29 01:30:00,948] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits +rank0 [2026-10-29 01:30:03,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits +rank0 [2026-10-29 01:30:04,267] INFO : Plotted number of elements +rank0 [2026-10-29 01:30:05,741] INFO : Plotted jets +rank0 [2026-10-29 01:30:07,390] INFO : Plotted jet ratio +rank0 [2026-10-29 01:33:55,391] INFO : Step 20100/50000 rank0 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2729 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1005 | Regression_eta: 0.0848 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0887 | LR: 6.86e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.3877s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank0 [2026-10-29 01:33:55,633] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:37:53,455] INFO : Step 20200/50000 rank0 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0662 | Regression_eta: 0.0630 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0652 | LR: 6.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5627s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank0 [2026-10-29 01:37:53,709] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:41:56,380] INFO : Step 20300/50000 rank0 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0729 | Regression_eta: 0.0596 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0954 | LR: 6.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4445s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank0 [2026-10-29 01:41:56,622] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:45:46,947] INFO : Step 20400/50000 rank0 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0824 | Regression_eta: 0.0514 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0638 | LR: 6.78e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.7862s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank0 [2026-10-29 01:45:47,189] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:49:33,652] INFO : Step 20500/50000 rank0 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2545 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0911 | Regression_eta: 0.0583 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0608 | LR: 6.76e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0420s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank0 [2026-10-29 01:49:33,892] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:53:32,813] INFO : Step 20600/50000 rank0 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0865 | Regression_eta: 0.0654 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0661 | LR: 6.73e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7536s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank0 [2026-10-29 01:53:33,060] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:57:30,608] INFO : Step 20700/50000 rank0 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0726 | Regression_eta: 0.0650 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0681 | LR: 6.70e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3275s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank0 [2026-10-29 01:57:30,849] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 59.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:01:36,203] INFO : Step 20800/50000 rank0 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2773 | Regression_cos_phi: 0.1008 | Regression_energy: 0.0872 | Regression_eta: 0.0795 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0840 | LR: 6.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.5771s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank0 [2026-10-29 02:01:36,447] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:05:50,263] INFO : Step 20900/50000 rank0 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0762 | Regression_eta: 0.0563 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0795 | LR: 6.65e-04 | DataLoad Time: 0.0186s | Model Forward Time: 6.7996s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank0 [2026-10-29 02:05:50,503] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 3% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 14% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 57.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 70% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:09:54,363] INFO : Step 21000/50000 rank0 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0910 | Regression_eta: 0.0550 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0768 | LR: 6.62e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5598s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank0 [2026-10-29 02:09:54,609] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:09:54,630] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-21000.pth +rank0 [2026-10-29 02:09:54,911] INFO : Running validation on rank0 +rank0 [2026-10-29 02:09:54,911] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:09:54,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION | Step=21000/50000 | Train Loss=1.3680 | Valid Loss=1.4534 | Train Loss Unweighted=0.7056 | Valid Loss Unweighted=0.7299 | Stale=3 | ETA=1239.3m +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 02:10:45,919] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari=0.39955 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.462257 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_competing=0.49808 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.581238 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.086177 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0504329 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:10:45,920] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/seconds_per_rank=21.879 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 02:10:45,921] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari=0.290006 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding_minus_raw/ari=0.159566 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 02:10:45,922] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:10:45,927] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:10:46,176] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:10:46,182] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 02:11:12,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:11:13,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:11:14,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:11:15,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:11:25,641] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:11:26,308] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:11:26,311] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:11:42,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:11:43,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:11:45,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:11:46,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:11:56,782] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 02:11:57,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:11:57,726] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 02:11:57,729] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 02:12:13,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:12:14,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:12:15,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:12:16,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:12:26,036] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 02:12:26,051] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:12:29,393] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:12:30,861] INFO : Plotted number of elements +rank0 [2026-10-29 02:12:32,484] INFO : Plotted jets +rank0 [2026-10-29 02:12:33,774] INFO : Plotted jet ratio +rank0 [2026-10-29 02:12:33,825] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:12:37,098] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:12:38,157] INFO : Plotted number of elements +rank0 [2026-10-29 02:12:39,758] INFO : Plotted jets +rank0 [2026-10-29 02:12:40,985] INFO : Plotted jet ratio +rank0 [2026-10-29 02:12:41,034] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits +rank0 [2026-10-29 02:12:43,575] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits +rank0 [2026-10-29 02:12:44,441] INFO : Plotted number of elements +rank0 [2026-10-29 02:12:46,365] INFO : Plotted jets +rank0 [2026-10-29 02:12:47,591] INFO : Plotted jet ratio +rank0 [2026-10-29 02:16:43,970] INFO : Step 21100/50000 rank0 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2543 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0841 | Regression_eta: 0.0615 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0859 | LR: 6.59e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4273s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank0 [2026-10-29 02:16:44,211] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 26% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 02:20:40,995] INFO : Step 21200/50000 rank0 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0853 | Regression_energy: 0.1176 | Regression_eta: 0.0654 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0576 | LR: 6.56e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7984s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank0 [2026-10-29 02:20:41,255] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:24:41,051] INFO : Step 21300/50000 rank0 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.2215 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0838 | Regression_eta: 0.0724 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0901 | LR: 6.54e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7769s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank0 [2026-10-29 02:24:41,298] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 6% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 98% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 3% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 98% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 1% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 73% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:28:36,031] INFO : Step 21400/50000 rank0 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.2357 | Regression_cos_phi: 0.1438 | Regression_energy: 0.0998 | Regression_eta: 0.1053 | Regression_pt: 0.1041 | Regression_sin_phi: 0.1255 | LR: 6.51e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.2099s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank0 [2026-10-29 02:28:36,285] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 3% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 89% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 55% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:32:40,114] INFO : Step 21500/50000 rank0 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0870 | Regression_eta: 0.0833 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0828 | LR: 6.48e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4488s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank0 [2026-10-29 02:32:40,389] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:36:35,839] INFO : Step 21600/50000 rank0 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0820 | Regression_eta: 0.0790 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0882 | LR: 6.45e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.5635s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank0 [2026-10-29 02:36:36,082] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:40:31,432] INFO : Step 21700/50000 rank0 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2973 | Regression_cos_phi: 0.1212 | Regression_energy: 0.0998 | Regression_eta: 0.1078 | Regression_pt: 0.1016 | Regression_sin_phi: 0.1387 | LR: 6.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.2798s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank0 [2026-10-29 02:40:31,673] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:44:21,471] INFO : Step 21800/50000 rank0 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0791 | Regression_energy: 0.1183 | Regression_eta: 0.0618 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0743 | LR: 6.40e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.9201s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank0 [2026-10-29 02:44:21,712] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 52.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:48:13,557] INFO : Step 21900/50000 rank0 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0865 | Regression_eta: 0.0877 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0932 | LR: 6.37e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9157s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank0 [2026-10-29 02:48:13,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:52:09,647] INFO : Step 22000/50000 rank0 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.2391 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0977 | Regression_eta: 0.0869 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0865 | LR: 6.34e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8922s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank0 [2026-10-29 02:52:09,889] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:52:09,911] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-22000.pth +rank0 [2026-10-29 02:52:10,088] INFO : Running validation on rank0 +rank0 [2026-10-29 02:52:10,088] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:52:10,088] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:52:58,538] INFO : VALIDATION | Step=22000/50000 | Train Loss=1.4853 | Valid Loss=1.3854 | Train Loss Unweighted=0.7623 | Valid Loss Unweighted=0.7132 | Stale=4 | ETA=1195.9m +rank0 [2026-10-29 02:52:58,538] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:52:58,538] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:52:58,538] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari=0.421664 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.495742 +rank0 [2026-10-29 02:52:58,539] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_competing=0.541627 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.593205 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0640634 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0624004 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 02:52:58,540] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/seconds_per_rank=21.7484 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari=0.290905 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding_minus_raw/ari=0.160466 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 02:52:58,541] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:52:58,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:52:58,808] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:52:58,812] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 02:53:25,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:53:27,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:53:28,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:53:29,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:53:39,880] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 02:53:40,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:53:40,600] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:53:40,604] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:53:56,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:53:57,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:53:59,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:54:00,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:54:10,159] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:54:10,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:54:10,679] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 02:54:10,682] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 02:54:26,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:54:27,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:54:27,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:54:28,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:54:38,549] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 02:54:38,577] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:54:41,896] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:54:43,048] INFO : Plotted number of elements +rank0 [2026-10-29 02:54:44,924] INFO : Plotted jets +rank0 [2026-10-29 02:54:46,097] INFO : Plotted jet ratio +rank0 [2026-10-29 02:54:46,153] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:54:49,359] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:54:50,385] INFO : Plotted number of elements +rank0 [2026-10-29 02:54:51,909] INFO : Plotted jets +rank0 [2026-10-29 02:54:53,084] INFO : Plotted jet ratio +rank0 [2026-10-29 02:54:53,134] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits +rank0 [2026-10-29 02:54:55,619] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits +rank0 [2026-10-29 02:54:56,470] INFO : Plotted number of elements +rank0 [2026-10-29 02:54:58,392] INFO : Plotted jets +rank0 [2026-10-29 02:54:59,566] INFO : Plotted jet ratio +rank0 [2026-10-29 02:58:47,064] INFO : Step 22100/50000 rank0 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2776 | Regression_cos_phi: 0.1026 | Regression_energy: 0.0914 | Regression_eta: 0.0942 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0985 | LR: 6.32e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.0516s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank0 [2026-10-29 02:58:47,304] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 59.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:02:37,614] INFO : Step 22200/50000 rank0 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0948 | Regression_eta: 0.1012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.1190 | LR: 6.29e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4689s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank0 [2026-10-29 03:02:37,862] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:06:29,494] INFO : Step 22300/50000 rank0 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2201 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0970 | Regression_eta: 0.0913 | Regression_pt: 0.1045 | Regression_sin_phi: 0.1070 | LR: 6.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5108s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank0 [2026-10-29 03:06:29,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 58.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:10:23,775] INFO : Step 22400/50000 rank0 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2837 | Regression_cos_phi: 0.0957 | Regression_energy: 0.1102 | Regression_eta: 0.0657 | Regression_pt: 0.1171 | Regression_sin_phi: 0.1094 | LR: 6.23e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2519s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank0 [2026-10-29 03:10:24,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:14:16,755] INFO : Step 22500/50000 rank0 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0952 | Regression_eta: 0.0546 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0811 | LR: 6.20e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5796s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank0 [2026-10-29 03:14:17,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:18:07,631] INFO : Step 22600/50000 rank0 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2517 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0992 | Regression_eta: 0.0771 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0886 | LR: 6.18e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2492s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank0 [2026-10-29 03:18:07,889] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:22:05,276] INFO : Step 22700/50000 rank0 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0961 | Regression_eta: 0.0615 | Regression_pt: 0.0994 | Regression_sin_phi: 0.1002 | LR: 6.15e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0145s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank0 [2026-10-29 03:22:05,517] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 58.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:26:07,726] INFO : Step 22800/50000 rank0 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2482 | Regression_cos_phi: 0.0978 | Regression_energy: 0.0949 | Regression_eta: 0.0703 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0832 | LR: 6.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1267s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank0 [2026-10-29 03:26:07,970] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 4% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 57.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 95% +6 10 0x7408, 62036 51.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 5% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:30:18,531] INFO : Step 22900/50000 rank0 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0833 | Regression_eta: 0.0496 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0659 | LR: 6.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.2715s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank0 [2026-10-29 03:30:18,804] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 57.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 50.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:34:04,481] INFO : Step 23000/50000 rank0 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.3099 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1170 | Regression_eta: 0.1049 | Regression_pt: 0.1234 | Regression_sin_phi: 0.1285 | LR: 6.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3898s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank0 [2026-10-29 03:34:04,741] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:34:04,762] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-23000.pth +rank0 [2026-10-29 03:34:04,954] INFO : Running validation on rank0 +rank0 [2026-10-29 03:34:04,954] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 03:34:04,954] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION | Step=23000/50000 | Train Loss=1.7099 | Valid Loss=1.6277 | Train Loss Unweighted=0.8922 | Valid Loss Unweighted=0.8502 | Stale=5 | ETA=1152.2m +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:34:52,976] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari=0.367829 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.450476 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_competing=0.489334 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.546873 +rank0 [2026-10-29 03:34:52,977] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.117898 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0160687 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/seconds_per_rank=21.8384 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 03:34:52,978] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari=0.273933 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding_minus_raw/ari=0.143493 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 03:34:52,979] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 03:34:52,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:34:53,241] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 03:34:53,245] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 03:35:19,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 03:35:20,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 03:35:21,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 03:35:22,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 03:35:32,414] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:35:32,426] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:35:32,951] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 03:35:32,955] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:35:49,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 03:35:50,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 03:35:51,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 03:35:52,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 03:36:02,366] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:36:02,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:36:02,911] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 03:36:02,914] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 03:36:18,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 03:36:18,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 03:36:19,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 03:36:20,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 03:36:30,305] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 03:36:30,933] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits +rank0 [2026-10-29 03:36:34,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits +rank0 [2026-10-29 03:36:35,649] INFO : Plotted number of elements +rank0 [2026-10-29 03:36:37,187] INFO : Plotted jets +rank0 [2026-10-29 03:36:38,393] INFO : Plotted jet ratio +rank0 [2026-10-29 03:36:38,469] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:36:41,617] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:36:42,597] INFO : Plotted number of elements +rank0 [2026-10-29 03:36:44,136] INFO : Plotted jets +rank0 [2026-10-29 03:36:45,312] INFO : Plotted jet ratio +rank0 [2026-10-29 03:36:45,382] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits +rank0 [2026-10-29 03:36:47,846] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits +rank0 [2026-10-29 03:36:48,707] INFO : Plotted number of elements +rank0 [2026-10-29 03:36:50,675] INFO : Plotted jets +rank0 [2026-10-29 03:36:51,900] INFO : Plotted jet ratio +rank0 [2026-10-29 03:40:33,852] INFO : Step 23100/50000 rank0 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2692 | Regression_cos_phi: 0.1224 | Regression_energy: 0.1021 | Regression_eta: 0.1069 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0863 | LR: 6.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1031s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank0 [2026-10-29 03:40:34,095] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:44:15,057] INFO : Step 23200/50000 rank0 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0917 | Regression_eta: 0.0711 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0689 | LR: 6.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1228s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank0 [2026-10-29 03:44:15,319] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:47:55,947] INFO : Step 23300/50000 rank0 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1288 | Regression_energy: 0.1022 | Regression_eta: 0.1122 | Regression_pt: 0.1088 | Regression_sin_phi: 0.1487 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6004s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank0 [2026-10-29 03:47:56,189] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:51:34,671] INFO : Step 23400/50000 rank0 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2359 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0848 | Regression_eta: 0.0488 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0840 | LR: 5.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0542s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank0 [2026-10-29 03:51:34,916] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 79% +4 8 0x7408, 13395 59.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:55:23,297] INFO : Step 23500/50000 rank0 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.2121 | Regression_cos_phi: 0.0773 | Regression_energy: 0.1100 | Regression_eta: 0.0648 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0801 | LR: 5.92e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6554s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank0 [2026-10-29 03:55:23,541] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 59.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:59:15,922] INFO : Step 23600/50000 rank0 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0945 | Regression_eta: 0.0551 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0583 | LR: 5.90e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6935s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank0 [2026-10-29 03:59:16,170] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:03:06,068] INFO : Step 23700/50000 rank0 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2640 | Regression_cos_phi: 0.1017 | Regression_energy: 0.0921 | Regression_eta: 0.0764 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0915 | LR: 5.87e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.0375s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank0 [2026-10-29 04:03:06,310] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 59.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 52% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:06:51,028] INFO : Step 23800/50000 rank0 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2637 | Regression_cos_phi: 0.1036 | Regression_energy: 0.0899 | Regression_eta: 0.1063 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0877 | LR: 5.84e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7080s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank0 [2026-10-29 04:06:51,291] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 59.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:11:05,385] INFO : Step 23900/50000 rank0 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1959 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0960 | Regression_eta: 0.0609 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0862 | LR: 5.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9898s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank0 [2026-10-29 04:11:05,626] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 57.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:14:58,405] INFO : Step 24000/50000 rank0 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1087 | Regression_energy: 0.0864 | Regression_eta: 0.0665 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0798 | LR: 5.78e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8280s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank0 [2026-10-29 04:14:58,649] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 1% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 98% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 59.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 88% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 5% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 6% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:14:58,814] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-24000.pth +rank0 [2026-10-29 04:14:59,048] INFO : Running validation on rank0 +rank0 [2026-10-29 04:14:59,048] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:14:59,049] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:15:48,452] INFO : VALIDATION | Step=24000/50000 | Train Loss=1.4637 | Valid Loss=1.4683 | Train Loss Unweighted=0.7560 | Valid Loss Unweighted=0.7609 | Stale=6 | ETA=1107.7m +rank0 [2026-10-29 04:15:48,452] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:15:48,452] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari=0.406988 +rank0 [2026-10-29 04:15:48,453] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.460233 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_competing=0.497419 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.560799 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0787394 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.029994 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 04:15:48,454] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/seconds_per_rank=22.0678 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari=0.272176 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding_minus_raw/ari=0.141736 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 04:15:48,455] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 04:15:48,456] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:15:48,461] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:15:48,720] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:15:48,724] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 04:16:15,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:16:16,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:16:17,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:16:18,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:16:28,918] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:16:29,442] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:16:29,446] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:16:45,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:16:46,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:16:47,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:16:48,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:16:58,657] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:16:59,583] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 04:16:59,586] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 04:17:15,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:17:16,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:17:16,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:17:17,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:17:27,567] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 04:17:27,583] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:17:30,853] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:17:31,958] INFO : Plotted number of elements +rank0 [2026-10-29 04:17:33,963] INFO : Plotted jets +rank0 [2026-10-29 04:17:35,192] INFO : Plotted jet ratio +rank0 [2026-10-29 04:17:35,258] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:17:38,456] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:17:39,477] INFO : Plotted number of elements +rank0 [2026-10-29 04:17:41,014] INFO : Plotted jets +rank0 [2026-10-29 04:17:42,189] INFO : Plotted jet ratio +rank0 [2026-10-29 04:17:42,239] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits +rank0 [2026-10-29 04:17:44,716] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits +rank0 [2026-10-29 04:17:45,577] INFO : Plotted number of elements +rank0 [2026-10-29 04:17:47,033] INFO : Plotted jets +rank0 [2026-10-29 04:17:48,691] INFO : Plotted jet ratio +rank0 [2026-10-29 04:21:44,723] INFO : Step 24100/50000 rank0 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2802 | Regression_cos_phi: 0.0977 | Regression_energy: 0.1027 | Regression_eta: 0.0871 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0932 | LR: 5.75e-04 | DataLoad Time: 0.0209s | Model Forward Time: 8.3980s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank0 [2026-10-29 04:21:44,988] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 57.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 50.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:25:34,742] INFO : Step 24200/50000 rank0 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0909 | Regression_energy: 0.1044 | Regression_eta: 0.0577 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0697 | LR: 5.73e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.1297s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank0 [2026-10-29 04:25:34,993] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:29:22,971] INFO : Step 24300/50000 rank0 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2033 | Regression_cos_phi: 0.1114 | Regression_energy: 0.1186 | Regression_eta: 0.0751 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0934 | LR: 5.70e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6903s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank0 [2026-10-29 04:29:23,225] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:33:08,235] INFO : Step 24400/50000 rank0 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2574 | Regression_cos_phi: 0.1166 | Regression_energy: 0.1380 | Regression_eta: 0.0804 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1152 | LR: 5.67e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank0 [2026-10-29 04:33:08,479] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 59.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:36:57,862] INFO : Step 24500/50000 rank0 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2369 | Regression_cos_phi: 0.0809 | Regression_energy: 0.1349 | Regression_eta: 0.0965 | Regression_pt: 0.1434 | Regression_sin_phi: 0.1119 | LR: 5.64e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9977s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank0 [2026-10-29 04:36:58,115] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:40:41,490] INFO : Step 24600/50000 rank0 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2075 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0897 | Regression_eta: 0.0620 | Regression_pt: 0.0928 | Regression_sin_phi: 0.1017 | LR: 5.61e-04 | DataLoad Time: 0.0253s | Model Forward Time: 4.8094s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank0 [2026-10-29 04:40:41,746] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 11% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 1% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:44:30,757] INFO : Step 24700/50000 rank0 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1044 | Classification_binary: 0.2716 | Regression_cos_phi: 0.1383 | Regression_energy: 0.1229 | Regression_eta: 0.1230 | Regression_pt: 0.1294 | Regression_sin_phi: 0.1221 | LR: 5.58e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2736s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank0 [2026-10-29 04:44:31,046] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:48:16,211] INFO : Step 24800/50000 rank0 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1006 | Regression_eta: 0.0794 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1098 | LR: 5.56e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.6495s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank0 [2026-10-29 04:48:16,521] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 2% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 88% +6 10 0x7408, 62036 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:52:01,325] INFO : Step 24900/50000 rank0 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0714 | Regression_energy: 0.1089 | Regression_eta: 0.0747 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0763 | LR: 5.53e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7055s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank0 [2026-10-29 04:52:01,608] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 59.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:55:45,019] INFO : Step 25000/50000 rank0 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1356 | Regression_energy: 0.1511 | Regression_eta: 0.1060 | Regression_pt: 0.1567 | Regression_sin_phi: 0.1043 | LR: 5.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9227s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank0 [2026-10-29 04:55:45,302] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:55:45,326] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-25000.pth +rank0 [2026-10-29 04:55:45,502] INFO : Running validation on rank0 +rank0 [2026-10-29 04:55:45,502] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:55:45,502] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:56:34,618] INFO : VALIDATION | Step=25000/50000 | Train Loss=1.6261 | Valid Loss=1.6229 | Train Loss Unweighted=0.8796 | Valid Loss Unweighted=0.8875 | Stale=7 | ETA=1063.2m +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 04:56:34,619] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari=0.402875 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.431398 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_competing=0.466371 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.557083 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.082852 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0262778 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 04:56:34,620] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/seconds_per_rank=22.0629 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari=0.265606 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 04:56:34,621] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding_minus_raw/ari=0.135167 +rank0 [2026-10-29 04:56:34,622] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 04:56:34,622] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 04:56:34,622] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:56:34,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:56:34,876] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:56:34,880] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 04:57:00,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:57:01,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:57:02,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:57:03,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:57:13,396] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 04:57:13,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:57:13,947] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:57:13,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:57:30,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:57:31,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:57:32,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:57:33,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:57:43,825] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:57:43,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:57:44,416] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 04:57:44,420] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 04:58:00,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:58:00,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:58:01,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:58:02,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:58:12,621] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 04:58:12,639] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:58:15,932] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:58:17,001] INFO : Plotted number of elements +rank0 [2026-10-29 04:58:18,959] INFO : Plotted jets +rank0 [2026-10-29 04:58:20,176] INFO : Plotted jet ratio +rank0 [2026-10-29 04:58:20,255] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:58:23,471] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:58:24,463] INFO : Plotted number of elements +rank0 [2026-10-29 04:58:26,001] INFO : Plotted jets +rank0 [2026-10-29 04:58:27,186] INFO : Plotted jet ratio +rank0 [2026-10-29 04:58:27,241] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits +rank0 [2026-10-29 04:58:29,728] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits +rank0 [2026-10-29 04:58:30,608] INFO : Plotted number of elements +rank0 [2026-10-29 04:58:32,143] INFO : Plotted jets +rank0 [2026-10-29 04:58:33,714] INFO : Plotted jet ratio +rank0 [2026-10-29 05:02:16,668] INFO : Step 25100/50000 rank0 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2729 | Regression_cos_phi: 0.1163 | Regression_energy: 0.1025 | Regression_eta: 0.0854 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0975 | LR: 5.47e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.4618s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank0 [2026-10-29 05:02:16,910] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 95% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 59.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:06:02,929] INFO : Step 25200/50000 rank0 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0897 | Regression_energy: 0.1081 | Regression_eta: 0.0794 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0752 | LR: 5.44e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1647s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank0 [2026-10-29 05:06:03,215] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:09:53,523] INFO : Step 25300/50000 rank0 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2029 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1086 | Regression_eta: 0.0605 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0803 | LR: 5.42e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5350s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank0 [2026-10-29 05:09:53,804] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:13:36,454] INFO : Step 25400/50000 rank0 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1024 | Classification_binary: 0.3133 | Regression_cos_phi: 0.1634 | Regression_energy: 0.1217 | Regression_eta: 0.0975 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0991 | LR: 5.39e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.8621s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank0 [2026-10-29 05:13:36,698] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 5% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 90% +6 10 0x7408, 62036 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:17:28,257] INFO : Step 25500/50000 rank0 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2406 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0946 | Regression_eta: 0.0626 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0635 | LR: 5.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7983s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank0 [2026-10-29 05:17:28,505] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:21:17,489] INFO : Step 25600/50000 rank0 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2278 | Regression_cos_phi: 0.1081 | Regression_energy: 0.1482 | Regression_eta: 0.1328 | Regression_pt: 0.1545 | Regression_sin_phi: 0.1031 | LR: 5.33e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6913s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank0 [2026-10-29 05:21:17,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:25:00,877] INFO : Step 25700/50000 rank0 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2670 | Regression_cos_phi: 0.1202 | Regression_energy: 0.1123 | Regression_eta: 0.0726 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1000 | LR: 5.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5600s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank0 [2026-10-29 05:25:01,122] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:28:45,395] INFO : Step 25800/50000 rank0 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.3315 | Regression_cos_phi: 0.1253 | Regression_energy: 0.1523 | Regression_eta: 0.0846 | Regression_pt: 0.1600 | Regression_sin_phi: 0.1237 | LR: 5.27e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1730s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank0 [2026-10-29 05:28:45,649] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:32:34,581] INFO : Step 25900/50000 rank0 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0901 | Regression_energy: 0.1091 | Regression_eta: 0.0928 | Regression_pt: 0.1131 | Regression_sin_phi: 0.1026 | LR: 5.25e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5438s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank0 [2026-10-29 05:32:34,827] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:36:29,305] INFO : Step 26000/50000 rank0 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2431 | Regression_cos_phi: 0.0811 | Regression_energy: 0.1123 | Regression_eta: 0.0927 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0904 | LR: 5.22e-04 | DataLoad Time: 0.0175s | Model Forward Time: 9.9025s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank0 [2026-10-29 05:36:29,551] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:36:29,573] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-26000.pth +rank0 [2026-10-29 05:36:29,856] INFO : Running validation on rank0 +rank0 [2026-10-29 05:36:29,856] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 05:36:29,856] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 05:37:20,129] INFO : VALIDATION | Step=26000/50000 | Train Loss=1.5398 | Valid Loss=1.5958 | Train Loss Unweighted=0.8285 | Valid Loss Unweighted=0.8408 | Stale=8 | ETA=1019.1m +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 05:37:20,130] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari=0.368806 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.44083 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_competing=0.476536 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.552034 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.116921 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0212291 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 05:37:20,131] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/seconds_per_rank=21.928 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari=0.278386 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 05:37:20,132] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding_minus_raw/ari=0.147946 +rank0 [2026-10-29 05:37:20,133] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 05:37:20,133] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 05:37:20,133] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 05:37:20,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:37:20,396] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 05:37:20,407] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 05:37:45,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 05:37:46,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 05:37:47,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 05:37:48,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 05:37:58,589] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 05:37:58,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:37:59,110] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 05:37:59,113] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:38:15,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 05:38:16,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 05:38:17,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 05:38:18,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 05:38:28,399] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 05:38:28,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:38:28,892] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 05:38:28,895] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 05:38:44,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 05:38:45,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 05:38:46,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 05:38:46,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 05:38:56,838] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 05:38:56,854] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits +rank0 [2026-10-29 05:39:00,196] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits +rank0 [2026-10-29 05:39:01,281] INFO : Plotted number of elements +rank0 [2026-10-29 05:39:02,866] INFO : Plotted jets +rank0 [2026-10-29 05:39:04,066] INFO : Plotted jet ratio +rank0 [2026-10-29 05:39:04,128] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:39:07,347] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:39:08,372] INFO : Plotted number of elements +rank0 [2026-10-29 05:39:10,311] INFO : Plotted jets +rank0 [2026-10-29 05:39:11,526] INFO : Plotted jet ratio +rank0 [2026-10-29 05:39:11,577] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits +rank0 [2026-10-29 05:39:13,919] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits +rank0 [2026-10-29 05:39:14,792] INFO : Plotted number of elements +rank0 [2026-10-29 05:39:16,322] INFO : Plotted jets +rank0 [2026-10-29 05:39:17,541] INFO : Plotted jet ratio +rank0 [2026-10-29 05:43:05,129] INFO : Step 26100/50000 rank0 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2805 | Regression_cos_phi: 0.1339 | Regression_energy: 0.1208 | Regression_eta: 0.0878 | Regression_pt: 0.1289 | Regression_sin_phi: 0.1342 | LR: 5.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5424s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank0 [2026-10-29 05:43:05,423] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 59.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:47:04,452] INFO : Step 26200/50000 rank0 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0882 | Regression_energy: 0.1069 | Regression_eta: 0.0604 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0768 | LR: 5.16e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.2137s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank0 [2026-10-29 05:47:04,700] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 6% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 98% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 17% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 1% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 98% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:50:53,323] INFO : Step 26300/50000 rank0 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0847 | Regression_energy: 0.1089 | Regression_eta: 0.0770 | Regression_pt: 0.1130 | Regression_sin_phi: 0.1006 | LR: 5.13e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.2200s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank0 [2026-10-29 05:50:53,567] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:54:53,416] INFO : Step 26400/50000 rank0 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.2926 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1428 | Regression_eta: 0.0843 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0988 | LR: 5.10e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.0994s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank0 [2026-10-29 05:54:53,664] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:58:42,842] INFO : Step 26500/50000 rank0 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.2393 | Regression_cos_phi: 0.0735 | Regression_energy: 0.1226 | Regression_eta: 0.0662 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0888 | LR: 5.08e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.1233s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank0 [2026-10-29 05:58:43,084] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:02:42,467] INFO : Step 26600/50000 rank0 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0708 | Regression_energy: 0.1244 | Regression_eta: 0.0611 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0828 | LR: 5.05e-04 | DataLoad Time: 0.0127s | Model Forward Time: 5.4065s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank0 [2026-10-29 06:02:42,722] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 57.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:06:40,343] INFO : Step 26700/50000 rank0 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2399 | Regression_cos_phi: 0.1848 | Regression_energy: 0.1131 | Regression_eta: 0.0859 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1133 | LR: 5.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.4026s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank0 [2026-10-29 06:06:40,585] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 57.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:10:41,934] INFO : Step 26800/50000 rank0 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2865 | Regression_cos_phi: 0.1332 | Regression_energy: 0.1317 | Regression_eta: 0.0972 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1291 | LR: 4.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9192s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank0 [2026-10-29 06:10:42,194] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:14:38,015] INFO : Step 26900/50000 rank0 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2384 | Regression_cos_phi: 0.1101 | Regression_energy: 0.1117 | Regression_eta: 0.0710 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0924 | LR: 4.96e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6584s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank0 [2026-10-29 06:14:38,258] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:18:43,676] INFO : Step 27000/50000 rank0 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.1627 | Regression_cos_phi: 0.1084 | Regression_energy: 0.0973 | Regression_eta: 0.0667 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0782 | LR: 4.94e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.6348s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank0 [2026-10-29 06:18:43,931] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:18:43,952] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-27000.pth +rank0 [2026-10-29 06:18:44,138] INFO : Running validation on rank0 +rank0 [2026-10-29 06:18:44,138] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 06:18:44,138] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION | Step=27000/50000 | Train Loss=1.4773 | Valid Loss=1.5213 | Train Loss Unweighted=0.7815 | Valid Loss Unweighted=0.8082 | Stale=9 | ETA=976.4m +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 06:19:33,285] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari=0.388535 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.414923 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_competing=0.441818 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.558499 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0971922 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0276939 +rank0 [2026-10-29 06:19:33,286] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/seconds_per_rank=21.9171 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 06:19:33,287] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari=0.263665 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding_minus_raw/ari=0.133225 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 06:19:33,288] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 06:19:33,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:19:33,566] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 06:19:33,572] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 06:20:00,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 06:20:01,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 06:20:03,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 06:20:04,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 06:20:14,214] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 06:20:14,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:20:14,782] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 06:20:14,786] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:20:31,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 06:20:33,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 06:20:34,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 06:20:35,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 06:20:45,782] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 06:20:45,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:20:46,292] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 06:20:46,296] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 06:21:02,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 06:21:02,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 06:21:03,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 06:21:04,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 06:21:14,406] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 06:21:14,426] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits +rank0 [2026-10-29 06:21:17,825] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits +rank0 [2026-10-29 06:21:18,867] INFO : Plotted number of elements +rank0 [2026-10-29 06:21:20,407] INFO : Plotted jets +rank0 [2026-10-29 06:21:21,943] INFO : Plotted jet ratio +rank0 [2026-10-29 06:21:21,995] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:21:25,233] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:21:26,223] INFO : Plotted number of elements +rank0 [2026-10-29 06:21:27,752] INFO : Plotted jets +rank0 [2026-10-29 06:21:28,931] INFO : Plotted jet ratio +rank0 [2026-10-29 06:21:28,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits +rank0 [2026-10-29 06:21:31,465] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits +rank0 [2026-10-29 06:21:32,317] INFO : Plotted number of elements +rank0 [2026-10-29 06:21:33,845] INFO : Plotted jets +rank0 [2026-10-29 06:21:35,078] INFO : Plotted jet ratio +rank0 [2026-10-29 06:25:29,443] INFO : Step 27100/50000 rank0 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2885 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1294 | Regression_eta: 0.0933 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0808 | LR: 4.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3052s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank0 [2026-10-29 06:25:29,705] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:29:30,247] INFO : Step 27200/50000 rank0 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1969 | Regression_cos_phi: 0.0568 | Regression_energy: 0.1036 | Regression_eta: 0.0640 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0728 | LR: 4.88e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.5233s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank0 [2026-10-29 06:29:30,494] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +2 6 0x7408, 15961 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:33:36,772] INFO : Step 27300/50000 rank0 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0891 | Regression_energy: 0.1053 | Regression_eta: 0.0901 | Regression_pt: 0.1114 | Regression_sin_phi: 0.1173 | LR: 4.85e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6472s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank0 [2026-10-29 06:33:37,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:37:37,432] INFO : Step 27400/50000 rank0 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2776 | Regression_cos_phi: 0.0973 | Regression_energy: 0.1069 | Regression_eta: 0.0849 | Regression_pt: 0.1125 | Regression_sin_phi: 0.1029 | LR: 4.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1991s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank0 [2026-10-29 06:37:37,716] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 57.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:41:29,674] INFO : Step 27500/50000 rank0 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.2997 | Regression_cos_phi: 0.1140 | Regression_energy: 0.1106 | Regression_eta: 0.0989 | Regression_pt: 0.1152 | Regression_sin_phi: 0.1214 | LR: 4.80e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.5979s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank0 [2026-10-29 06:41:29,916] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:45:24,055] INFO : Step 27600/50000 rank0 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1198 | Regression_eta: 0.0704 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0932 | LR: 4.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5329s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank0 [2026-10-29 06:45:24,313] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:49:16,715] INFO : Step 27700/50000 rank0 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.2001 | Regression_cos_phi: 0.0783 | Regression_energy: 0.1251 | Regression_eta: 0.0543 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0806 | LR: 4.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7235s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank0 [2026-10-29 06:49:16,962] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 3% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 1% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:53:16,794] INFO : Step 27800/50000 rank0 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2690 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1144 | Regression_eta: 0.0944 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0830 | LR: 4.71e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.3158s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank0 [2026-10-29 06:53:17,039] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 6% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 12% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:57:18,718] INFO : Step 27900/50000 rank0 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0703 | Regression_energy: 0.1149 | Regression_eta: 0.0835 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0771 | LR: 4.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2929s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank0 [2026-10-29 06:57:18,997] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:01:10,903] INFO : Step 28000/50000 rank0 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0790 | Regression_energy: 0.1112 | Regression_eta: 0.0529 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0895 | LR: 4.66e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.9416s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank0 [2026-10-29 07:01:11,146] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 93% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 82% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:01:11,171] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-28000.pth +rank0 [2026-10-29 07:01:11,418] INFO : Running validation on rank0 +rank0 [2026-10-29 07:01:11,418] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:01:11,418] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:02:02,306] INFO : VALIDATION | Step=28000/50000 | Train Loss=1.4297 | Valid Loss=1.5173 | Train Loss Unweighted=0.7873 | Valid Loss Unweighted=0.8211 | Stale=10 | ETA=934.0m +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 07:02:02,307] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari=0.427067 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.505323 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_competing=0.555285 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.596726 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0586599 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0659214 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:02:02,308] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/seconds_per_rank=22.22 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:02:02,309] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari=0.244065 +rank0 [2026-10-29 07:02:02,310] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 07:02:02,310] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding_minus_raw/ari=0.113625 +rank0 [2026-10-29 07:02:02,310] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 07:02:02,310] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 07:02:02,310] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:02:02,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:02:02,675] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:02:02,687] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 07:02:30,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:02:31,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:02:32,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:02:33,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:02:43,534] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:02:44,046] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:02:44,050] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:03:00,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:03:01,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:03:02,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:03:03,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:03:13,613] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:03:14,471] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 07:03:14,475] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 07:03:30,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:03:30,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:03:31,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:03:32,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:03:42,329] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 07:03:42,346] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:03:45,614] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:03:46,680] INFO : Plotted number of elements +rank0 [2026-10-29 07:03:48,285] INFO : Plotted jets +rank0 [2026-10-29 07:03:49,576] INFO : Plotted jet ratio +rank0 [2026-10-29 07:03:49,648] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:03:52,823] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:03:53,815] INFO : Plotted number of elements +rank0 [2026-10-29 07:03:55,724] INFO : Plotted jets +rank0 [2026-10-29 07:03:56,897] INFO : Plotted jet ratio +rank0 [2026-10-29 07:03:56,948] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits +rank0 [2026-10-29 07:03:59,240] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits +rank0 [2026-10-29 07:04:00,103] INFO : Plotted number of elements +rank0 [2026-10-29 07:04:01,633] INFO : Plotted jets +rank0 [2026-10-29 07:04:02,864] INFO : Plotted jet ratio +rank0 [2026-10-29 07:08:05,244] INFO : Step 28100/50000 rank0 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2310 | Regression_cos_phi: 0.0969 | Regression_energy: 0.1172 | Regression_eta: 0.0898 | Regression_pt: 0.1209 | Regression_sin_phi: 0.1034 | LR: 4.63e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.8136s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank0 [2026-10-29 07:08:05,485] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 24% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 57.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 31% 0% +6 10 0x7408, 62036 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:11:57,255] INFO : Step 28200/50000 rank0 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0967 | Regression_energy: 0.1157 | Regression_eta: 0.0649 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0855 | LR: 4.60e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8763s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank0 [2026-10-29 07:11:57,514] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:16:11,384] INFO : Step 28300/50000 rank0 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0776 | Regression_energy: 0.1276 | Regression_eta: 0.0583 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0787 | LR: 4.57e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4757s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank0 [2026-10-29 07:16:11,626] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 3% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 1% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 57.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:20:04,575] INFO : Step 28400/50000 rank0 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2394 | Regression_cos_phi: 0.1337 | Regression_energy: 0.1237 | Regression_eta: 0.0860 | Regression_pt: 0.1291 | Regression_sin_phi: 0.1183 | LR: 4.55e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7583s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank0 [2026-10-29 07:20:04,858] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:24:00,868] INFO : Step 28500/50000 rank0 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2784 | Regression_cos_phi: 0.0805 | Regression_energy: 0.1188 | Regression_eta: 0.0827 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0951 | LR: 4.52e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.7970s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank0 [2026-10-29 07:24:01,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:27:53,194] INFO : Step 28600/50000 rank0 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0699 | Regression_energy: 0.1145 | Regression_eta: 0.0674 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0910 | LR: 4.49e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9462s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank0 [2026-10-29 07:27:53,435] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:31:49,874] INFO : Step 28700/50000 rank0 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0913 | Regression_energy: 0.1081 | Regression_eta: 0.0762 | Regression_pt: 0.1125 | Regression_sin_phi: 0.1019 | LR: 4.46e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.7951s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank0 [2026-10-29 07:31:50,115] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:35:45,355] INFO : Step 28800/50000 rank0 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2202 | Regression_cos_phi: 0.0957 | Regression_energy: 0.1056 | Regression_eta: 0.0626 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0605 | LR: 4.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1328s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank0 [2026-10-29 07:35:45,598] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:39:39,631] INFO : Step 28900/50000 rank0 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1934 | Regression_cos_phi: 0.1402 | Regression_energy: 0.1091 | Regression_eta: 0.0586 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0846 | LR: 4.41e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.1605s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank0 [2026-10-29 07:39:39,892] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:43:44,574] INFO : Step 29000/50000 rank0 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0903 | Regression_eta: 0.0541 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0566 | LR: 4.38e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6150s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank0 [2026-10-29 07:43:44,817] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:43:44,843] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-29000.pth +rank0 [2026-10-29 07:43:45,030] INFO : Running validation on rank0 +rank0 [2026-10-29 07:43:45,030] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:43:45,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION | Step=29000/50000 | Train Loss=1.3528 | Valid Loss=1.4464 | Train Loss Unweighted=0.7183 | Valid Loss Unweighted=0.7622 | Stale=11 | ETA=891.6m +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 07:44:33,748] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari=0.401123 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.526982 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_competing=0.57987 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.571653 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.084604 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0408478 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:44:33,749] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/seconds_per_rank=21.999 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 07:44:33,750] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari=0.288258 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding_minus_raw/ari=0.157818 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 07:44:33,751] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:44:33,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:44:34,014] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:44:34,025] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 07:44:59,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:45:00,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:45:01,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:45:02,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:45:12,859] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 07:45:12,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:45:13,371] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:45:13,375] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:45:29,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:45:30,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:45:31,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:45:33,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:45:43,221] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:45:43,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:45:43,719] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 07:45:43,722] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 07:45:59,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:46:00,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:46:00,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:46:01,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:46:11,628] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 07:46:11,645] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:46:14,838] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:46:15,889] INFO : Plotted number of elements +rank0 [2026-10-29 07:46:17,451] INFO : Plotted jets +rank0 [2026-10-29 07:46:18,586] INFO : Plotted jet ratio +rank0 [2026-10-29 07:46:18,727] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:46:21,991] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:46:23,001] INFO : Plotted number of elements +rank0 [2026-10-29 07:46:24,888] INFO : Plotted jets +rank0 [2026-10-29 07:46:26,070] INFO : Plotted jet ratio +rank0 [2026-10-29 07:46:26,126] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits +rank0 [2026-10-29 07:46:28,399] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits +rank0 [2026-10-29 07:46:29,255] INFO : Plotted number of elements +rank0 [2026-10-29 07:46:30,769] INFO : Plotted jets +rank0 [2026-10-29 07:46:31,926] INFO : Plotted jet ratio +rank0 [2026-10-29 07:50:42,814] INFO : Step 29100/50000 rank0 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2265 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1247 | Regression_eta: 0.0591 | Regression_pt: 0.1299 | Regression_sin_phi: 0.1017 | LR: 4.35e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.3737s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank0 [2026-10-29 07:50:43,099] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 57.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:54:31,185] INFO : Step 29200/50000 rank0 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0974 | Regression_eta: 0.0863 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1202 | LR: 4.33e-04 | DataLoad Time: 0.0186s | Model Forward Time: 6.0568s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank0 [2026-10-29 07:54:31,433] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:58:23,538] INFO : Step 29300/50000 rank0 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0863 | Regression_eta: 0.0433 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0572 | LR: 4.30e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.6514s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank0 [2026-10-29 07:58:23,780] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 92% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:02:12,121] INFO : Step 29400/50000 rank0 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2092 | Regression_cos_phi: 0.0951 | Regression_energy: 0.1221 | Regression_eta: 0.0705 | Regression_pt: 0.1276 | Regression_sin_phi: 0.1153 | LR: 4.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2314s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank0 [2026-10-29 08:02:12,364] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:05:55,466] INFO : Step 29500/50000 rank0 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2317 | Regression_cos_phi: 0.0973 | Regression_energy: 0.0939 | Regression_eta: 0.0568 | Regression_pt: 0.0984 | Regression_sin_phi: 0.1007 | LR: 4.24e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.2310s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank0 [2026-10-29 08:05:55,709] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 59.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:09:50,289] INFO : Step 29600/50000 rank0 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0977 | Regression_eta: 0.0761 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0793 | LR: 4.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3722s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank0 [2026-10-29 08:09:50,570] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:13:32,404] INFO : Step 29700/50000 rank0 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0935 | Regression_eta: 0.0596 | Regression_pt: 0.0975 | Regression_sin_phi: 0.1013 | LR: 4.19e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.9305s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank0 [2026-10-29 08:13:32,651] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:17:25,524] INFO : Step 29800/50000 rank0 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0906 | Regression_eta: 0.0820 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0905 | LR: 4.16e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.1204s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank0 [2026-10-29 08:17:25,778] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:21:11,416] INFO : Step 29900/50000 rank0 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0757 | Regression_energy: 0.1123 | Regression_eta: 0.0764 | Regression_pt: 0.1178 | Regression_sin_phi: 0.1223 | LR: 4.14e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.8567s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank0 [2026-10-29 08:21:11,663] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 7% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 31% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 98% +6 10 0x7408, 62036 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:24:57,046] INFO : Step 30000/50000 rank0 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0964 | Regression_eta: 0.0609 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0863 | LR: 4.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.3627s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank0 [2026-10-29 08:24:57,290] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:24:57,360] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-30000.pth +rank0 [2026-10-29 08:24:57,566] INFO : Running validation on rank0 +rank0 [2026-10-29 08:24:57,566] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 08:24:57,566] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 08:25:45,669] INFO : VALIDATION | Step=30000/50000 | Train Loss=1.3076 | Valid Loss=1.3791 | Train Loss Unweighted=0.6729 | Valid Loss Unweighted=0.7096 | Stale=12 | ETA=848.3m +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 08:25:45,670] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari=0.434783 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.52612 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_competing=0.577186 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.598307 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0509446 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0675026 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 08:25:45,671] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/seconds_per_rank=21.916 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari=0.277775 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding_minus_raw/ari=0.147335 +rank0 [2026-10-29 08:25:45,672] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 08:25:45,673] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 08:25:45,673] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 08:25:45,677] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:25:45,936] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 08:25:45,941] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 08:26:11,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 08:26:12,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 08:26:13,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 08:26:14,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 08:26:24,807] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 08:26:24,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:26:25,384] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 08:26:25,387] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:26:41,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 08:26:42,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 08:26:43,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 08:26:44,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 08:26:54,558] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 08:26:54,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:26:55,054] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 08:26:55,057] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 08:27:10,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 08:27:11,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 08:27:12,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 08:27:13,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 08:27:23,093] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 08:27:23,129] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits +rank0 [2026-10-29 08:27:26,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits +rank0 [2026-10-29 08:27:27,517] INFO : Plotted number of elements +rank0 [2026-10-29 08:27:29,111] INFO : Plotted jets +rank0 [2026-10-29 08:27:30,280] INFO : Plotted jet ratio +rank0 [2026-10-29 08:27:30,335] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:27:33,489] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:27:34,506] INFO : Plotted number of elements +rank0 [2026-10-29 08:27:36,485] INFO : Plotted jets +rank0 [2026-10-29 08:27:37,669] INFO : Plotted jet ratio +rank0 [2026-10-29 08:27:37,726] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits +rank0 [2026-10-29 08:27:40,035] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits +rank0 [2026-10-29 08:27:40,939] INFO : Plotted number of elements +rank0 [2026-10-29 08:27:42,596] INFO : Plotted jets +rank0 [2026-10-29 08:27:43,854] INFO : Plotted jet ratio +rank0 [2026-10-29 08:31:31,863] INFO : Step 30100/50000 rank0 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0987 | Regression_energy: 0.1031 | Regression_eta: 0.0826 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0800 | LR: 4.08e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5031s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank0 [2026-10-29 08:31:32,151] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 26% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 100% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:35:24,558] INFO : Step 30200/50000 rank0 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.2327 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0988 | Regression_eta: 0.0554 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0714 | LR: 4.06e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.1851s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank0 [2026-10-29 08:35:24,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 100% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:39:31,175] INFO : Step 30300/50000 rank0 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0861 | Regression_eta: 0.0634 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0680 | LR: 4.03e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.9008s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank0 [2026-10-29 08:39:31,418] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:43:19,844] INFO : Step 30400/50000 rank0 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2268 | Regression_cos_phi: 0.1058 | Regression_energy: 0.1029 | Regression_eta: 0.0813 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0682 | LR: 4.00e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1835s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank0 [2026-10-29 08:43:20,124] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:47:04,354] INFO : Step 30500/50000 rank0 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2368 | Regression_cos_phi: 0.0798 | Regression_energy: 0.1078 | Regression_eta: 0.0665 | Regression_pt: 0.1129 | Regression_sin_phi: 0.1262 | LR: 3.98e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.5345s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank0 [2026-10-29 08:47:04,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:50:51,368] INFO : Step 30600/50000 rank0 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0927 | Regression_eta: 0.1095 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0770 | LR: 3.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2351s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank0 [2026-10-29 08:50:51,661] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:54:36,519] INFO : Step 30700/50000 rank0 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1846 | Regression_cos_phi: 0.1152 | Regression_energy: 0.0865 | Regression_eta: 0.0563 | Regression_pt: 0.0896 | Regression_sin_phi: 0.1004 | LR: 3.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1514s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank0 [2026-10-29 08:54:36,779] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:58:21,649] INFO : Step 30800/50000 rank0 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2648 | Regression_cos_phi: 0.1106 | Regression_energy: 0.1035 | Regression_eta: 0.0899 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1045 | LR: 3.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.9091s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank0 [2026-10-29 08:58:21,921] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:02:10,700] INFO : Step 30900/50000 rank0 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2025 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0850 | Regression_eta: 0.0473 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0588 | LR: 3.87e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.8304s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank0 [2026-10-29 09:02:10,944] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 98% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 58.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 91% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:05:53,107] INFO : Step 31000/50000 rank0 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0690 | Regression_eta: 0.0423 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0491 | LR: 3.84e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.9272s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank0 [2026-10-29 09:05:53,350] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 8% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 2% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 92% +4 8 0x7408, 13395 59.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 0% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:05:53,372] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-31000.pth +rank0 [2026-10-29 09:05:53,672] INFO : Running validation on rank0 +rank0 [2026-10-29 09:05:53,672] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:05:53,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION | Step=31000/50000 | Train Loss=1.1995 | Valid Loss=1.3051 | Train Loss Unweighted=0.6113 | Valid Loss Unweighted=0.6593 | Stale=13 | ETA=805.0m +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 09:06:42,235] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari=0.411675 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.560135 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_competing=0.623926 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.606 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0740527 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0751956 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:06:42,236] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/seconds_per_rank=21.9655 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:06:42,237] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari=0.290558 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding_minus_raw/ari=0.160118 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 09:06:42,238] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:06:42,243] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:06:42,513] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:06:42,524] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 09:07:08,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:07:09,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:07:10,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:07:12,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:07:22,081] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 09:07:22,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:07:22,582] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:07:22,586] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:07:38,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:07:39,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:07:40,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:07:41,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:07:51,970] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 09:07:51,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:07:52,510] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 09:07:52,514] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 09:08:08,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:08:09,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:08:09,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:08:10,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:08:20,543] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 09:08:20,706] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:08:23,970] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:08:25,061] INFO : Plotted number of elements +rank0 [2026-10-29 09:08:26,609] INFO : Plotted jets +rank0 [2026-10-29 09:08:27,769] INFO : Plotted jet ratio +rank0 [2026-10-29 09:08:27,842] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:08:30,994] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:08:32,022] INFO : Plotted number of elements +rank0 [2026-10-29 09:08:34,013] INFO : Plotted jets +rank0 [2026-10-29 09:08:35,226] INFO : Plotted jet ratio +rank0 [2026-10-29 09:08:35,284] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits +rank0 [2026-10-29 09:08:37,595] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits +rank0 [2026-10-29 09:08:38,447] INFO : Plotted number of elements +rank0 [2026-10-29 09:08:39,988] INFO : Plotted jets +rank0 [2026-10-29 09:08:41,173] INFO : Plotted jet ratio +rank0 [2026-10-29 09:12:28,806] INFO : Step 31100/50000 rank0 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2426 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0931 | Regression_eta: 0.0763 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0785 | LR: 3.82e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.4043s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank0 [2026-10-29 09:12:29,051] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 58.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:16:29,917] INFO : Step 31200/50000 rank0 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2459 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0843 | Regression_eta: 0.0653 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0684 | LR: 3.79e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2447s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank0 [2026-10-29 09:16:30,163] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:20:25,642] INFO : Step 31300/50000 rank0 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0709 | Regression_energy: 0.1011 | Regression_eta: 0.0866 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0611 | LR: 3.76e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.2778s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank0 [2026-10-29 09:20:25,884] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:24:21,085] INFO : Step 31400/50000 rank0 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2195 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0846 | Regression_eta: 0.1031 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0825 | LR: 3.74e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5379s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank0 [2026-10-29 09:24:21,329] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:28:17,938] INFO : Step 31500/50000 rank0 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2373 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0939 | Regression_eta: 0.1070 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0740 | LR: 3.71e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.1197s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank0 [2026-10-29 09:28:18,182] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:32:17,484] INFO : Step 31600/50000 rank0 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0857 | Regression_eta: 0.0374 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0595 | LR: 3.69e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.5492s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank0 [2026-10-29 09:32:17,729] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:36:10,031] INFO : Step 31700/50000 rank0 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0908 | Regression_eta: 0.0580 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0892 | LR: 3.66e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.8675s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank0 [2026-10-29 09:36:10,276] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:40:23,312] INFO : Step 31800/50000 rank0 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0788 | Regression_eta: 0.0859 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0868 | LR: 3.64e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0423s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank0 [2026-10-29 09:40:23,557] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:48:16,100] INFO : Step 31900/50000 rank0 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0806 | Regression_eta: 0.0511 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0775 | LR: 3.61e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.6222s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank0 [2026-10-29 09:48:16,347] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 57.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 65% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:52:26,527] INFO : Step 32000/50000 rank0 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0780 | Regression_eta: 0.0590 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0590 | LR: 3.58e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7888s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank0 [2026-10-29 09:52:26,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 58.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:52:26,848] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-32000.pth +rank0 [2026-10-29 09:52:27,028] INFO : Running validation on rank0 +rank0 [2026-10-29 09:52:27,028] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:52:27,028] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:53:17,986] INFO : VALIDATION | Step=32000/50000 | Train Loss=1.2067 | Valid Loss=1.2863 | Train Loss Unweighted=0.6180 | Valid Loss Unweighted=0.6412 | Stale=0 | ETA=765.0m +rank0 [2026-10-29 09:53:17,986] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:53:17,986] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari=0.425934 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 09:53:17,987] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.579667 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_competing=0.64286 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.596446 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0597937 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0656409 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 09:53:17,988] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/seconds_per_rank=21.9464 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari=0.293342 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding_minus_raw/ari=0.162902 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 09:53:17,989] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:53:17,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:53:18,254] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:53:18,266] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 09:53:46,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:53:47,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:53:49,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:53:50,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:54:00,494] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:54:01,009] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:54:01,013] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:54:16,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:54:17,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:54:18,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:54:20,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:54:30,057] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:54:30,792] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 09:54:30,795] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 09:54:46,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:54:47,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:54:47,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:54:48,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:54:58,597] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 09:54:58,730] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:55:02,074] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:55:03,171] INFO : Plotted number of elements +rank0 [2026-10-29 09:55:04,836] INFO : Plotted jets +rank0 [2026-10-29 09:55:06,046] INFO : Plotted jet ratio +rank0 [2026-10-29 09:55:06,096] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:55:09,392] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:55:10,452] INFO : Plotted number of elements +rank0 [2026-10-29 09:55:12,413] INFO : Plotted jets +rank0 [2026-10-29 09:55:13,601] INFO : Plotted jet ratio +rank0 [2026-10-29 09:55:13,651] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits +rank0 [2026-10-29 09:55:15,956] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits +rank0 [2026-10-29 09:55:16,820] INFO : Plotted number of elements +rank0 [2026-10-29 09:55:18,390] INFO : Plotted jets +rank0 [2026-10-29 09:55:19,640] INFO : Plotted jet ratio +rank0 [2026-10-29 09:59:20,176] INFO : Step 32100/50000 rank0 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0915 | Regression_eta: 0.0414 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0715 | LR: 3.56e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.8794s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank0 [2026-10-29 09:59:20,419] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 57.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:03:27,124] INFO : Step 32200/50000 rank0 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0731 | Regression_eta: 0.0490 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0537 | LR: 3.53e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1883s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank0 [2026-10-29 10:03:27,366] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:07:23,532] INFO : Step 32300/50000 rank0 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1818 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0974 | Regression_eta: 0.0462 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0819 | LR: 3.51e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3260s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank0 [2026-10-29 10:07:23,775] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:11:43,664] INFO : Step 32400/50000 rank0 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0696 | Regression_eta: 0.0471 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0636 | LR: 3.48e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.6953s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank0 [2026-10-29 10:11:43,927] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 2% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 5% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 2% +4 8 0x7408, 13395 57.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 98% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:15:57,310] INFO : Step 32500/50000 rank0 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2171 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0851 | Regression_eta: 0.0573 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0792 | LR: 3.46e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.4667s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank0 [2026-10-29 10:15:57,559] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 58.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:20:13,921] INFO : Step 32600/50000 rank0 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0794 | Regression_eta: 0.0625 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0729 | LR: 3.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1269s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank0 [2026-10-29 10:20:14,165] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:24:12,439] INFO : Step 32700/50000 rank0 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0727 | Regression_eta: 0.0704 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0866 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7262s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank0 [2026-10-29 10:24:12,702] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:28:06,502] INFO : Step 32800/50000 rank0 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2231 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0816 | Regression_eta: 0.0660 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0735 | LR: 3.38e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2435s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank0 [2026-10-29 10:28:06,749] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:32:14,930] INFO : Step 32900/50000 rank0 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0891 | Regression_eta: 0.0536 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0654 | LR: 3.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.5834s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank0 [2026-10-29 10:32:15,173] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 8% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 87% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 97% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:36:15,695] INFO : Step 33000/50000 rank0 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0748 | Regression_eta: 0.0496 | Regression_pt: 0.0774 | Regression_sin_phi: 0.1085 | LR: 3.33e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5246s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank0 [2026-10-29 10:36:15,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 57.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:36:15,982] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-33000.pth +rank0 [2026-10-29 10:36:16,162] INFO : Running validation on rank0 +rank0 [2026-10-29 10:36:16,162] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 10:36:16,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION | Step=33000/50000 | Train Loss=1.2030 | Valid Loss=1.3062 | Train Loss Unweighted=0.5934 | Valid Loss Unweighted=0.6524 | Stale=1 | ETA=723.1m +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 10:37:04,347] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari=0.391588 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.578331 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_competing=0.642017 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.608705 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0941395 +rank0 [2026-10-29 10:37:04,348] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0779002 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/seconds_per_rank=22.2656 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 10:37:04,349] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari=0.278346 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding_minus_raw/ari=0.147907 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 10:37:04,350] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 10:37:04,356] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:37:04,627] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 10:37:04,636] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 10:37:32,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 10:37:34,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 10:37:36,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 10:37:38,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 10:37:48,234] INFO : Time taken to make predictions on device 0 is: 0.31 min +rank0 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:37:48,815] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 10:37:48,818] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:38:05,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 10:38:06,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 10:38:07,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 10:38:08,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 10:38:18,582] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:38:21,199] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:38:21,709] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 10:38:21,713] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 10:38:37,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 10:38:37,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 10:38:38,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 10:38:39,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 10:38:49,574] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 10:38:49,818] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits +rank0 [2026-10-29 10:38:53,356] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits +rank0 [2026-10-29 10:38:54,454] INFO : Plotted number of elements +rank0 [2026-10-29 10:38:56,078] INFO : Plotted jets +rank0 [2026-10-29 10:38:57,240] INFO : Plotted jet ratio +rank0 [2026-10-29 10:38:57,325] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:39:00,592] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:39:01,610] INFO : Plotted number of elements +rank0 [2026-10-29 10:39:03,598] INFO : Plotted jets +rank0 [2026-10-29 10:39:04,799] INFO : Plotted jet ratio +rank0 [2026-10-29 10:39:04,858] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits +rank0 [2026-10-29 10:39:07,236] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits +rank0 [2026-10-29 10:39:08,104] INFO : Plotted number of elements +rank0 [2026-10-29 10:39:09,663] INFO : Plotted jets +rank0 [2026-10-29 10:39:10,863] INFO : Plotted jet ratio +rank0 [2026-10-29 10:43:12,900] INFO : Step 33100/50000 rank0 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2202 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0711 | Regression_eta: 0.0618 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0747 | LR: 3.31e-04 | DataLoad Time: 0.0119s | Model Forward Time: 5.8028s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank0 [2026-10-29 10:43:13,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 23% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:47:23,543] INFO : Step 33200/50000 rank0 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2378 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0928 | Regression_eta: 0.0936 | Regression_pt: 0.0995 | Regression_sin_phi: 0.1020 | LR: 3.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3963s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank0 [2026-10-29 10:47:23,787] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 25% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:51:33,974] INFO : Step 33300/50000 rank0 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1868 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0839 | Regression_eta: 0.0595 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0704 | LR: 3.26e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.2110s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank0 [2026-10-29 10:51:34,219] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 25% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:55:48,606] INFO : Step 33400/50000 rank0 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0894 | Regression_eta: 0.0424 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0830 | LR: 3.23e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3581s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank0 [2026-10-29 10:55:48,881] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 12% +2 6 0x7408, 15961 50.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 35% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 97% +4 8 0x7408, 13395 57.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 10% +6 10 0x7408, 62036 50.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 25% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:59:49,344] INFO : Step 33500/50000 rank0 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2060 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0899 | Regression_eta: 0.0600 | Regression_pt: 0.0957 | Regression_sin_phi: 0.1000 | LR: 3.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.6354s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank0 [2026-10-29 10:59:49,610] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 26% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:05:25,199] INFO : Step 33600/50000 rank0 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2281 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0891 | Regression_eta: 0.0623 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0932 | LR: 3.19e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.9024s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank0 [2026-10-29 11:05:25,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 26% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:09:09,384] INFO : Step 33700/50000 rank0 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0649 | Regression_eta: 0.0393 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0601 | LR: 3.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0217s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank0 [2026-10-29 11:09:09,649] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 26% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:13:03,022] INFO : Step 33800/50000 rank0 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0506 | Regression_energy: 0.1021 | Regression_eta: 0.0598 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0963 | LR: 3.14e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1009s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank0 [2026-10-29 11:13:03,268] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 1% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 2% +4 8 0x7408, 13395 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 96% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:16:53,229] INFO : Step 33900/50000 rank0 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0711 | Regression_eta: 0.0476 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0673 | LR: 3.11e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.5011s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank0 [2026-10-29 11:16:53,475] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:20:48,523] INFO : Step 34000/50000 rank0 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0785 | Regression_eta: 0.0802 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0915 | LR: 3.09e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.6844s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank0 [2026-10-29 11:20:48,768] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 58.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:20:48,873] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-34000.pth +rank0 [2026-10-29 11:20:49,055] INFO : Running validation on rank0 +rank0 [2026-10-29 11:20:49,055] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 11:20:49,055] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 11:21:42,927] INFO : VALIDATION | Step=34000/50000 | Train Loss=1.1988 | Valid Loss=1.2503 | Train Loss Unweighted=0.5891 | Valid Loss Unweighted=0.6295 | Stale=0 | ETA=681.6m +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 11:21:42,928] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari=0.419275 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.538653 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_competing=0.590655 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.590312 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0664525 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0595074 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 11:21:42,929] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/seconds_per_rank=22.1351 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari=0.287191 +rank0 [2026-10-29 11:21:42,930] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 11:21:42,931] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding_minus_raw/ari=0.156752 +rank0 [2026-10-29 11:21:42,931] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 11:21:42,931] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 11:21:42,931] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 11:21:42,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:21:43,194] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 11:21:43,200] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 11:22:09,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 11:22:10,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 11:22:11,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 11:22:12,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 11:22:22,498] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:22:22,977] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 11:22:22,981] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:22:39,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 11:22:40,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 11:22:41,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 11:22:42,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 11:22:52,926] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 11:22:52,937] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:22:53,478] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 11:22:53,482] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 11:23:09,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 11:23:10,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 11:23:11,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 11:23:11,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 11:23:21,930] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 11:23:21,948] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits +rank0 [2026-10-29 11:23:25,279] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits +rank0 [2026-10-29 11:23:26,365] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:27,969] INFO : Plotted jets +rank0 [2026-10-29 11:23:29,138] INFO : Plotted jet ratio +rank0 [2026-10-29 11:23:29,212] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:23:32,443] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:23:33,450] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:35,006] INFO : Plotted jets +rank0 [2026-10-29 11:23:36,221] INFO : Plotted jet ratio +rank0 [2026-10-29 11:23:36,272] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits +rank0 [2026-10-29 11:23:38,759] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits +rank0 [2026-10-29 11:23:40,107] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:41,611] INFO : Plotted jets +rank0 [2026-10-29 11:23:42,823] INFO : Plotted jet ratio +rank0 [2026-10-29 11:27:43,428] INFO : Step 34100/50000 rank0 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0812 | Regression_eta: 0.0575 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0817 | LR: 3.06e-04 | DataLoad Time: 0.0172s | Model Forward Time: 8.3529s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank0 [2026-10-29 11:27:43,692] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 56.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +6 10 0x7408, 62036 51.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:31:45,971] INFO : Step 34200/50000 rank0 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2351 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0851 | Regression_eta: 0.0707 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0928 | LR: 3.04e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4330s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank0 [2026-10-29 11:31:46,228] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:35:57,502] INFO : Step 34300/50000 rank0 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0992 | Regression_energy: 0.0766 | Regression_eta: 0.0667 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0945 | LR: 3.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 7.9300s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank0 [2026-10-29 11:35:57,782] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 1% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 44% +4 8 0x7408, 13395 57.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 31% 6% +6 10 0x7408, 62036 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:40:09,059] INFO : Step 34400/50000 rank0 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0731 | Regression_energy: 0.1000 | Regression_eta: 0.0486 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0638 | LR: 2.99e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.5969s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank0 [2026-10-29 11:40:09,345] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:44:05,521] INFO : Step 34500/50000 rank0 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2235 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0843 | Regression_eta: 0.0617 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0907 | LR: 2.97e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.2055s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank0 [2026-10-29 11:44:05,768] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:47:59,606] INFO : Step 34600/50000 rank0 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0715 | Regression_eta: 0.0620 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0779 | LR: 2.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3380s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank0 [2026-10-29 11:47:59,850] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 57.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 96% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 92% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:52:00,826] INFO : Step 34700/50000 rank0 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0647 | Regression_eta: 0.0442 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0521 | LR: 2.92e-04 | DataLoad Time: 0.0221s | Model Forward Time: 6.3122s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank0 [2026-10-29 11:52:01,070] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 24% +2 6 0x7408, 15961 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 91% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:55:54,655] INFO : Step 34800/50000 rank0 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0720 | Regression_eta: 0.0368 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0410 | LR: 2.90e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5983s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank0 [2026-10-29 11:55:54,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 59.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:59:59,450] INFO : Step 34900/50000 rank0 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0895 | Regression_eta: 0.0572 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0695 | LR: 2.88e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9101s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank0 [2026-10-29 11:59:59,697] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 58.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:04:07,036] INFO : Step 35000/50000 rank0 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0907 | Regression_eta: 0.0526 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0671 | LR: 2.85e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1752s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank0 [2026-10-29 12:04:07,282] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:04:07,368] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 12:04:07,554] INFO : Running validation on rank0 +rank0 [2026-10-29 12:04:07,554] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 12:04:07,554] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 12:05:05,421] INFO : VALIDATION | Step=35000/50000 | Train Loss=1.2809 | Valid Loss=1.2680 | Train Loss Unweighted=0.6303 | Valid Loss Unweighted=0.6390 | Stale=1 | ETA=639.3m +rank0 [2026-10-29 12:05:05,421] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari=0.434364 +rank0 [2026-10-29 12:05:05,422] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.562777 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_competing=0.625183 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.60667 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0513637 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0758653 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 12:05:05,423] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/seconds_per_rank=21.871 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari=0.284593 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding_minus_raw/ari=0.154153 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 12:05:05,424] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 12:05:05,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:05:05,685] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 12:05:05,691] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 12:05:32,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 12:05:33,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 12:05:34,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 12:05:35,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 12:05:45,353] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:05:45,912] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 12:05:45,915] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:06:02,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 12:06:03,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 12:06:04,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 12:06:05,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 12:06:15,501] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 12:06:15,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:06:16,076] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 12:06:16,080] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 12:06:31,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 12:06:32,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 12:06:33,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 12:06:33,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 12:06:43,871] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 12:06:44,249] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits +rank0 [2026-10-29 12:06:47,513] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits +rank0 [2026-10-29 12:06:48,632] INFO : Plotted number of elements +rank0 [2026-10-29 12:06:50,259] INFO : Plotted jets +rank0 [2026-10-29 12:06:51,503] INFO : Plotted jet ratio +rank0 [2026-10-29 12:06:51,555] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:06:54,768] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:06:55,851] INFO : Plotted number of elements +rank0 [2026-10-29 12:06:57,495] INFO : Plotted jets +rank0 [2026-10-29 12:06:58,757] INFO : Plotted jet ratio +rank0 [2026-10-29 12:06:58,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits +rank0 [2026-10-29 12:07:01,655] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits +rank0 [2026-10-29 12:07:02,447] INFO : Plotted number of elements +rank0 [2026-10-29 12:07:04,029] INFO : Plotted jets +rank0 [2026-10-29 12:07:05,244] INFO : Plotted jet ratio +rank0 [2026-10-29 12:11:07,971] INFO : Step 35100/50000 rank0 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1049 | Regression_eta: 0.0552 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0555 | LR: 2.83e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.5617s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank0 [2026-10-29 12:11:08,234] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 57.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 50.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 99% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:15:06,858] INFO : Step 35200/50000 rank0 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0754 | Regression_eta: 0.0442 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0547 | LR: 2.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.7959s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank0 [2026-10-29 12:15:07,105] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +2 6 0x7408, 15961 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +4 8 0x7408, 13395 58.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 70% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 93% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 29% 0% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:18:58,117] INFO : Step 35300/50000 rank0 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2137 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0973 | Regression_eta: 0.0609 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0979 | LR: 2.79e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.4877s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank0 [2026-10-29 12:18:58,359] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:23:00,861] INFO : Step 35400/50000 rank0 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0858 | Regression_eta: 0.0469 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0603 | LR: 2.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0519s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank0 [2026-10-29 12:23:01,107] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:27:15,120] INFO : Step 35500/50000 rank0 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0835 | Regression_eta: 0.0379 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0698 | LR: 2.74e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9131s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank0 [2026-10-29 12:27:15,366] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 58.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:31:18,130] INFO : Step 35600/50000 rank0 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0794 | Regression_eta: 0.0468 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0457 | LR: 2.72e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.9525s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank0 [2026-10-29 12:31:18,374] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 57.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:35:31,913] INFO : Step 35700/50000 rank0 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1893 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0858 | Regression_eta: 0.0841 | Regression_pt: 0.0899 | Regression_sin_phi: 0.1079 | LR: 2.70e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9344s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank0 [2026-10-29 12:47:12,923] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 43.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 44.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 44.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 45.0°C 118.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:49:26,342] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-29 14:49:26,352] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank0 [2026-10-29 14:49:53,254] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-29 14:49:53,254] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank0 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank0 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank0 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank0 [2026-10-29 14:49:53,279] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,283] INFO : Initializing backbone layer 3 +rank0 [2026-10-29 14:49:53,283] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,286] INFO : Initializing backbone layer 4 +rank0 [2026-10-29 14:49:53,286] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,289] INFO : Initializing backbone layer 5 +rank0 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank0 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank0 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank0 [2026-10-29 14:49:53,342] INFO : Output DNNs initialization took 0.05s +rank0 [2026-10-29 14:49:53,342] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-29 14:49:53,342] INFO : jagged_attention enabled=True +rank0 [2026-10-29 14:49:53,342] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-29 14:49:53,342] INFO : task_queries enabled=False +rank0 [2026-10-29 14:49:53,342] INFO : classification_norm parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : regression_norm parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : classification_query parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank0 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank0 [2026-10-29 14:49:53,343] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-29 14:49:53,343] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_pid parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_pt parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_eta parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-29 14:49:53,343] INFO : nn_energy parameters: 0 +rank0 [2026-10-29 14:49:53,344] INFO : Total MLPF parameters: 7260244 +rank0 [2026-10-29 14:49:53,344] INFO : MLPF __init__ done +rank0 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank0 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=0 +rank0 [2026-10-29 14:49:53,363] INFO : Moved model to device rank=0 +rank0 [2026-10-29 14:49:53,364] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-29 14:49:53,692] INFO : Restoring training from step 35001 +rank0 [2026-10-29 14:49:53,698] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 14:49:53,698] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-29 14:49:53,928] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-29 14:49:54,044] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank0 [2026-10-29 14:49:54,044] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-29 14:49:54,045] INFO : Trainable parameters: 7260244 +rank0 [2026-10-29 14:49:54,045] INFO : Non-trainable parameters: 0 +rank0 [2026-10-29 14:49:54,046] INFO : Total parameters: 7260244 +rank0 [2026-10-29 14:49:54,048] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-29 14:49:54,067] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851 +rank0 [2026-10-29 14:49:54,067] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851 +rank0 [2026-10-29 14:49:54,067] INFO : Getting dataloaders +rank0 [2026-10-29 14:49:54,067] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:00,256] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,357] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,463] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,563] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,666] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,748] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,146] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,227] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,308] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,389] INFO : train_dataset: clic_edm_ttbar_hits, 90900 +rank0 [2026-10-29 14:50:01,389] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:01,469] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,564] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,649] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,750] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,841] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,942] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,038] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,140] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,229] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,342] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-29 14:50:02,342] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:02,438] INFO : train_dataset: clic_edm_qq_hits, 90000 +rank0 [2026-10-29 14:50:02,782] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-29 14:50:02,858] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-29 14:50:02,938] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-29 14:50:03,039] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-29 14:50:03,133] INFO : train_dataset: clic_edm_qq_hits, 89996 +rank0 [2026-10-29 14:50:03,228] INFO : train_dataset: clic_edm_qq_hits, 89998 +rank0 [2026-10-29 14:50:03,339] INFO : train_dataset: clic_edm_qq_hits, 89994 +rank0 [2026-10-29 14:50:03,449] INFO : train_dataset: clic_edm_qq_hits, 89995 +rank0 [2026-10-29 14:50:03,546] INFO : train_dataset: clic_edm_qq_hits, 90099 +rank0 [2026-10-29 14:50:03,546] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-29 14:50:03,546] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-29 14:50:03,546] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-29 14:50:03,546] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:03,575] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,601] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,627] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,654] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,680] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,706] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,734] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,761] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,791] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,817] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,817] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:03,844] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,868] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,894] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,919] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,945] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,970] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,995] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,020] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,045] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,071] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,071] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:04,096] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,121] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,146] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,171] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,198] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,224] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,339] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,366] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,391] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,417] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,417] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-29 14:50:04,417] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-29 14:50:18,309] INFO : Got dataloaders +rank0 [2026-10-29 14:50:18,310] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank0 [2026-10-29 14:50:18,310] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank0 [2026-10-29 14:50:18,310] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank0 [2026-10-29 14:50:18,310] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank0 [2026-10-29 14:50:20,061] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-29 14:50:20,062] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-29 14:50:20,062] INFO : Starting training from step 35001 to 50000 on rank 0 of 8 +rank0 [2026-10-29 14:50:20,067] INFO : Creating train_iterator +rank0 [2026-10-29 14:50:20,067] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank0 [2026-10-29 14:50:34,159] INFO : Created train_iterator +rank0 [2026-10-29 14:54:16,963] INFO : Step 35100/50000 rank0 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0855 | Regression_eta: 0.0544 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0555 | LR: 2.83e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.4666s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank0 [2026-10-29 14:54:17,202] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 197.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 50.0°C 197.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:57:54,327] INFO : Step 35200/50000 rank0 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0702 | Regression_eta: 0.0454 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0555 | LR: 2.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1502s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank0 [2026-10-29 14:57:54,565] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:01:33,877] INFO : Step 35300/50000 rank0 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2075 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0987 | Regression_eta: 0.0580 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0905 | LR: 2.79e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3141s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank0 [2026-10-29 15:01:34,113] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 45.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 49.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:05:11,313] INFO : Step 35400/50000 rank0 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0802 | Regression_eta: 0.0462 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0713 | LR: 2.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0680s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank0 [2026-10-29 15:05:11,549] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:08:49,964] INFO : Step 35500/50000 rank0 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0746 | Regression_eta: 0.0360 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0742 | LR: 2.74e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1026s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank0 [2026-10-29 15:08:50,205] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:12:28,095] INFO : Step 35600/50000 rank0 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0747 | Regression_eta: 0.0484 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0622 | LR: 2.72e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.3641s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank0 [2026-10-29 15:12:28,335] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 46.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:16:08,008] INFO : Step 35700/50000 rank0 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.1849 | Regression_cos_phi: 0.1179 | Regression_energy: 0.0902 | Regression_eta: 0.0833 | Regression_pt: 0.0951 | Regression_sin_phi: 0.1190 | LR: 2.70e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9740s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank0 [2026-10-29 15:16:08,243] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:19:48,293] INFO : Step 35800/50000 rank0 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0801 | Regression_eta: 0.0561 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0604 | LR: 2.68e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1637s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank0 [2026-10-29 15:19:48,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:23:31,332] INFO : Step 35900/50000 rank0 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2140 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0941 | Regression_eta: 0.0757 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0981 | LR: 2.65e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5396s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank0 [2026-10-29 15:23:31,570] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:27:19,822] INFO : Step 36000/50000 rank0 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0727 | Regression_eta: 0.0672 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0661 | LR: 2.63e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4673s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank0 [2026-10-29 15:27:20,061] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:27:20,135] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-36000.pth +rank0 [2026-10-29 15:27:20,332] INFO : Running validation on rank0 +rank0 [2026-10-29 15:27:20,332] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 15:27:20,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 15:28:08,915] INFO : VALIDATION | Step=36000/50000 | Train Loss=1.2206 | Valid Loss=1.2266 | Train Loss Unweighted=0.6064 | Valid Loss Unweighted=0.6124 | Stale=0 | ETA=14.7m +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 15:28:08,916] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari=0.40544 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.560833 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_competing=0.619569 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.60051 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.080287 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0697049 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 15:28:08,917] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/seconds_per_rank=22.2152 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari=0.288943 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding_minus_raw/ari=0.158503 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 15:28:08,918] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 15:28:08,919] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 15:28:08,924] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:28:09,249] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 15:28:09,269] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 15:28:36,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 15:28:37,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 15:28:38,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 15:28:39,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 15:28:49,940] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 15:28:49,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:28:50,427] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 15:28:50,430] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:29:06,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 15:29:07,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 15:29:08,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 15:29:09,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 15:29:19,468] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 15:29:19,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:29:20,197] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 15:29:20,199] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 15:29:36,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 15:29:37,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 15:29:38,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 15:29:38,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 15:29:48,752] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 15:29:48,768] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits +rank0 [2026-10-29 15:29:52,032] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits +rank0 [2026-10-29 15:29:53,379] INFO : Plotted number of elements +rank0 [2026-10-29 15:29:55,000] INFO : Plotted jets +rank0 [2026-10-29 15:29:56,437] INFO : Plotted jet ratio +rank0 [2026-10-29 15:29:56,499] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:29:59,833] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:30:00,823] INFO : Plotted number of elements +rank0 [2026-10-29 15:30:02,335] INFO : Plotted jets +rank0 [2026-10-29 15:30:03,514] INFO : Plotted jet ratio +rank0 [2026-10-29 15:30:03,565] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits +rank0 [2026-10-29 15:30:06,011] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits +rank0 [2026-10-29 15:30:06,871] INFO : Plotted number of elements +rank0 [2026-10-29 15:30:08,414] INFO : Plotted jets +rank0 [2026-10-29 15:30:10,015] INFO : Plotted jet ratio +rank0 [2026-10-29 15:33:57,207] INFO : Step 36100/50000 rank0 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0755 | Regression_eta: 0.0518 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0625 | LR: 2.61e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5661s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank0 [2026-10-29 15:33:57,443] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:37:36,698] INFO : Step 36200/50000 rank0 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2460 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0819 | Regression_eta: 0.0686 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0931 | LR: 2.59e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2445s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank0 [2026-10-29 15:37:36,940] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +6 10 0x7408, 62036 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:41:20,979] INFO : Step 36300/50000 rank0 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.2216 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0695 | Regression_eta: 0.0884 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0654 | LR: 2.57e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1575s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank0 [2026-10-29 15:41:21,218] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 184.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:45:13,474] INFO : Step 36400/50000 rank0 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1621 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0984 | Regression_eta: 0.0452 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0655 | LR: 2.55e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9099s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank0 [2026-10-29 15:45:13,712] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 2% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 6% +2 6 0x7408, 15961 45.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 16% +4 8 0x7408, 13395 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:49:02,676] INFO : Step 36500/50000 rank0 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0994 | Regression_energy: 0.0717 | Regression_eta: 0.0623 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0888 | LR: 2.52e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6821s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank0 [2026-10-29 15:49:02,915] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:52:52,595] INFO : Step 36600/50000 rank0 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2064 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0946 | Regression_eta: 0.0609 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0583 | LR: 2.50e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3754s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank0 [2026-10-29 15:52:52,831] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 46.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:56:51,999] INFO : Step 36700/50000 rank0 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0831 | Regression_eta: 0.0545 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0670 | LR: 2.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4830s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank0 [2026-10-29 15:56:52,237] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:00:44,175] INFO : Step 36800/50000 rank0 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0764 | Regression_eta: 0.0692 | Regression_pt: 0.0796 | Regression_sin_phi: 0.1023 | LR: 2.46e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0668s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank0 [2026-10-29 16:00:44,414] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 46.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:04:40,516] INFO : Step 36900/50000 rank0 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1022 | Regression_eta: 0.0649 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0823 | LR: 2.44e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4981s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank0 [2026-10-29 16:04:40,753] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:08:30,783] INFO : Step 37000/50000 rank0 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2090 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0836 | Regression_eta: 0.0856 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0778 | LR: 2.42e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5419s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank0 [2026-10-29 16:08:31,020] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:08:31,044] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-37000.pth +rank0 [2026-10-29 16:08:31,223] INFO : Running validation on rank0 +rank0 [2026-10-29 16:08:31,223] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:08:31,223] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION | Step=37000/50000 | Train Loss=1.2519 | Valid Loss=1.2612 | Train Loss Unweighted=0.6356 | Valid Loss Unweighted=0.6289 | Stale=1 | ETA=27.8m +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 16:09:20,810] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari=0.425804 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.534915 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_competing=0.58689 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.584826 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0599236 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.054021 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 16:09:20,811] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/seconds_per_rank=21.8467 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:09:20,812] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari=0.280151 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding_minus_raw/ari=0.149711 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 16:09:20,813] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:09:20,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:09:21,073] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:09:21,078] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 16:09:46,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:09:47,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:09:48,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:09:49,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:09:59,726] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:10:00,305] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:10:00,309] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:10:16,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:10:17,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:10:18,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:10:19,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:10:29,924] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:10:30,500] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 16:10:30,503] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 16:10:46,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:10:46,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:10:47,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:10:48,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:10:58,332] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 16:10:58,492] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:11:01,806] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:11:02,983] INFO : Plotted number of elements +rank0 [2026-10-29 16:11:04,549] INFO : Plotted jets +rank0 [2026-10-29 16:11:05,682] INFO : Plotted jet ratio +rank0 [2026-10-29 16:11:05,765] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:11:09,011] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:11:10,011] INFO : Plotted number of elements +rank0 [2026-10-29 16:11:11,519] INFO : Plotted jets +rank0 [2026-10-29 16:11:12,952] INFO : Plotted jet ratio +rank0 [2026-10-29 16:11:13,025] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits +rank0 [2026-10-29 16:11:15,257] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits +rank0 [2026-10-29 16:11:16,008] INFO : Plotted number of elements +rank0 [2026-10-29 16:11:17,512] INFO : Plotted jets +rank0 [2026-10-29 16:11:18,665] INFO : Plotted jet ratio +rank0 [2026-10-29 16:15:07,066] INFO : Step 37100/50000 rank0 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0770 | Regression_eta: 0.0445 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0631 | LR: 2.40e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5111s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank0 [2026-10-29 16:15:07,305] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:18:46,964] INFO : Step 37200/50000 rank0 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0815 | Regression_eta: 0.0433 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0691 | LR: 2.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9091s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank0 [2026-10-29 16:18:47,206] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:22:17,393] INFO : Step 37300/50000 rank0 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2161 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0811 | Regression_eta: 0.0522 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0641 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2458s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank0 [2026-10-29 16:22:17,640] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:26:02,481] INFO : Step 37400/50000 rank0 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0873 | Regression_eta: 0.0558 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0701 | LR: 2.34e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5965s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank0 [2026-10-29 16:26:49,208] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 46.0°C 105.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 40.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 42.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 48.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:30:28,544] INFO : Step 37500/50000 rank0 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0833 | Regression_eta: 0.0512 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0798 | LR: 2.32e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2538s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank0 [2026-10-29 16:30:28,798] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:34:15,583] INFO : Step 37600/50000 rank0 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2341 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0873 | Regression_eta: 0.0672 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0823 | LR: 2.30e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7539s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank0 [2026-10-29 16:34:15,819] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:38:01,512] INFO : Step 37700/50000 rank0 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0929 | Regression_eta: 0.0769 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0720 | LR: 2.28e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8698s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank0 [2026-10-29 16:38:01,759] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 5% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 98% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:41:52,087] INFO : Step 37800/50000 rank0 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0855 | Regression_eta: 0.0465 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0536 | LR: 2.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9681s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank0 [2026-10-29 16:41:52,325] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:45:40,513] INFO : Step 37900/50000 rank0 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2033 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0745 | Regression_eta: 0.0648 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0777 | LR: 2.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5071s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank0 [2026-10-29 16:45:40,749] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:49:32,897] INFO : Step 38000/50000 rank0 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0734 | Regression_eta: 0.0938 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0922 | LR: 2.22e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5919s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank0 [2026-10-29 16:49:33,134] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:49:33,156] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-38000.pth +rank0 [2026-10-29 16:49:33,477] INFO : Running validation on rank0 +rank0 [2026-10-29 16:49:33,477] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:49:33,477] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION | Step=38000/50000 | Train Loss=1.2294 | Valid Loss=1.2483 | Train Loss Unweighted=0.6043 | Valid Loss Unweighted=0.6348 | Stale=2 | ETA=37.9m +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 16:50:21,396] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari=0.428368 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.539725 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_competing=0.59706 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.603345 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.057359 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0725406 +rank0 [2026-10-29 16:50:21,397] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/seconds_per_rank=21.8471 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 16:50:21,398] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari=0.286841 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding_minus_raw/ari=0.156401 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 16:50:21,399] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:50:21,405] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:50:21,658] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:50:21,679] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 16:50:47,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:50:49,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:50:52,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:50:56,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:51:06,225] INFO : Time taken to make predictions on device 0 is: 0.34 min +rank0 [2026-10-29 16:51:06,244] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:51:06,743] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:51:06,747] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:51:23,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:51:24,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:51:25,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:51:26,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:51:36,376] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:51:36,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:51:36,911] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 16:51:36,914] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 16:51:52,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:51:53,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:51:54,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:51:54,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:52:04,911] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 16:52:05,223] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:52:08,417] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:52:09,749] INFO : Plotted number of elements +rank0 [2026-10-29 16:52:11,227] INFO : Plotted jets +rank0 [2026-10-29 16:52:12,359] INFO : Plotted jet ratio +rank0 [2026-10-29 16:52:12,443] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:52:15,673] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:52:16,619] INFO : Plotted number of elements +rank0 [2026-10-29 16:52:18,124] INFO : Plotted jets +rank0 [2026-10-29 16:52:19,261] INFO : Plotted jet ratio +rank0 [2026-10-29 16:52:19,329] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits +rank0 [2026-10-29 16:52:21,841] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits +rank0 [2026-10-29 16:52:22,700] INFO : Plotted number of elements +rank0 [2026-10-29 16:52:24,595] INFO : Plotted jets +rank0 [2026-10-29 16:52:25,779] INFO : Plotted jet ratio +rank0 [2026-10-29 16:56:14,183] INFO : Step 38100/50000 rank0 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0704 | Regression_eta: 0.0429 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0575 | LR: 2.20e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.0237s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank0 [2026-10-29 16:56:14,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:00:01,704] INFO : Step 38200/50000 rank0 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0929 | Regression_eta: 0.0635 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0892 | LR: 2.18e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5203s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank0 [2026-10-29 17:00:01,945] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:03:45,731] INFO : Step 38300/50000 rank0 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.2386 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0792 | Regression_eta: 0.0548 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0825 | LR: 2.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9687s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank0 [2026-10-29 17:03:45,968] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:07:29,625] INFO : Step 38400/50000 rank0 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0599 | Regression_energy: 0.1038 | Regression_eta: 0.0617 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0604 | LR: 2.14e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2531s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank0 [2026-10-29 17:07:29,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:11:17,895] INFO : Step 38500/50000 rank0 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1419 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0844 | Regression_eta: 0.0487 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0606 | LR: 2.12e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2871s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank0 [2026-10-29 17:11:18,135] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:15:07,179] INFO : Step 38600/50000 rank0 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.2147 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0635 | Regression_eta: 0.0515 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0758 | LR: 2.11e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.0386s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank0 [2026-10-29 17:15:07,414] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:18:55,681] INFO : Step 38700/50000 rank0 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0742 | Regression_eta: 0.0405 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0419 | LR: 2.09e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.5250s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank0 [2026-10-29 17:18:55,920] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:22:45,028] INFO : Step 38800/50000 rank0 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0797 | Regression_eta: 0.0381 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0523 | LR: 2.07e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2000s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank0 [2026-10-29 17:22:45,269] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:26:36,573] INFO : Step 38900/50000 rank0 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1917 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0687 | Regression_eta: 0.0654 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0732 | LR: 2.05e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.3721s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank0 [2026-10-29 17:26:36,810] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:30:47,098] INFO : Step 39000/50000 rank0 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0740 | Regression_eta: 0.0384 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0435 | LR: 2.03e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.3029s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank0 [2026-10-29 17:30:47,336] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 45.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 53.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:30:47,463] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-39000.pth +rank0 [2026-10-29 17:30:47,644] INFO : Running validation on rank0 +rank0 [2026-10-29 17:30:47,644] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 17:30:47,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 17:31:36,192] INFO : VALIDATION | Step=39000/50000 | Train Loss=1.2081 | Valid Loss=1.1987 | Train Loss Unweighted=0.5935 | Valid Loss Unweighted=0.5988 | Stale=0 | ETA=45.5m +rank0 [2026-10-29 17:31:36,192] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:31:36,192] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 17:31:36,192] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 17:31:36,192] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari=0.374446 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.573681 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_competing=0.632785 +rank0 [2026-10-29 17:31:36,193] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.603674 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.111281 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0728688 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 17:31:36,194] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/seconds_per_rank=22.0322 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari=0.306662 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding_minus_raw/ari=0.176222 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 17:31:36,195] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 17:31:36,201] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:31:36,456] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 17:31:36,466] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 17:32:01,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 17:32:03,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 17:32:04,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 17:32:05,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 17:32:15,332] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:32:15,855] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 17:32:15,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:32:32,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 17:32:33,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 17:32:34,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 17:32:35,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 17:32:45,358] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 17:32:45,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:32:45,862] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 17:32:45,870] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 17:33:01,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 17:33:02,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 17:33:02,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 17:33:03,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 17:33:13,636] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 17:33:13,652] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits +rank0 [2026-10-29 17:33:16,859] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits +rank0 [2026-10-29 17:33:17,910] INFO : Plotted number of elements +rank0 [2026-10-29 17:33:19,679] INFO : Plotted jets +rank0 [2026-10-29 17:33:20,835] INFO : Plotted jet ratio +rank0 [2026-10-29 17:33:20,903] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:33:24,104] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:33:25,098] INFO : Plotted number of elements +rank0 [2026-10-29 17:33:26,642] INFO : Plotted jets +rank0 [2026-10-29 17:33:27,810] INFO : Plotted jet ratio +rank0 [2026-10-29 17:33:27,862] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits +rank0 [2026-10-29 17:33:30,313] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits +rank0 [2026-10-29 17:33:31,160] INFO : Plotted number of elements +rank0 [2026-10-29 17:33:32,946] INFO : Plotted jets +rank0 [2026-10-29 17:33:34,114] INFO : Plotted jet ratio +rank0 [2026-10-29 17:37:23,666] INFO : Step 39100/50000 rank0 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0911 | Regression_eta: 0.0522 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0820 | LR: 2.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4572s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank0 [2026-10-29 17:37:23,904] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:41:18,737] INFO : Step 39200/50000 rank0 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0935 | Regression_eta: 0.0705 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0798 | LR: 2.00e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3780s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank0 [2026-10-29 17:41:18,989] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:45:06,408] INFO : Step 39300/50000 rank0 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2292 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0844 | Regression_eta: 0.0555 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0826 | LR: 1.98e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5973s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank0 [2026-10-29 17:45:06,645] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:49:01,940] INFO : Step 39400/50000 rank0 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0885 | Regression_eta: 0.0574 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0573 | LR: 1.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7717s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank0 [2026-10-29 17:49:02,178] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:53:07,009] INFO : Step 39500/50000 rank0 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0882 | Regression_eta: 0.0527 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0579 | LR: 1.94e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0027s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank0 [2026-10-29 17:53:07,257] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 21% +4 8 0x7408, 13395 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:56:59,724] INFO : Step 39600/50000 rank0 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0866 | Regression_eta: 0.0492 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0828 | LR: 1.93e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.8749s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank0 [2026-10-29 17:56:59,970] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:00:44,198] INFO : Step 39700/50000 rank0 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0830 | Regression_eta: 0.0554 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0646 | LR: 1.91e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8830s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank0 [2026-10-29 18:00:44,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:04:32,736] INFO : Step 39800/50000 rank0 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0724 | Regression_eta: 0.0403 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0568 | LR: 1.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5930s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank0 [2026-10-29 18:04:32,973] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 5% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:08:20,106] INFO : Step 39900/50000 rank0 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.2231 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0746 | Regression_eta: 0.0708 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0869 | LR: 1.88e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8411s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank0 [2026-10-29 18:08:20,357] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:12:06,084] INFO : Step 40000/50000 rank0 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0772 | Regression_eta: 0.0572 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0626 | LR: 1.86e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5908s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank0 [2026-10-29 18:12:06,325] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:12:06,346] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-40000.pth +rank0 [2026-10-29 18:12:06,524] INFO : Running validation on rank0 +rank0 [2026-10-29 18:12:06,524] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:12:06,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:12:59,649] INFO : VALIDATION | Step=40000/50000 | Train Loss=1.2693 | Valid Loss=1.2382 | Train Loss Unweighted=0.6322 | Valid Loss Unweighted=0.6143 | Stale=1 | ETA=50.7m +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 18:12:59,650] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari=0.431694 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.562164 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_competing=0.620185 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.59381 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0540331 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0630054 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 18:12:59,651] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/seconds_per_rank=21.9529 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari=0.282112 +rank0 [2026-10-29 18:12:59,652] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 18:12:59,653] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding_minus_raw/ari=0.151672 +rank0 [2026-10-29 18:12:59,653] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 18:12:59,653] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 18:12:59,653] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:12:59,659] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:12:59,909] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:12:59,913] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 18:13:23,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:13:24,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:13:26,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:13:27,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:13:37,165] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:13:37,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:13:37,720] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:13:37,724] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:13:53,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:13:54,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:13:55,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:13:56,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:14:06,829] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:14:07,478] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 18:14:07,482] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 18:14:23,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:14:24,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:14:24,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:14:25,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:14:35,765] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 18:14:35,863] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:14:39,117] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:14:40,195] INFO : Plotted number of elements +rank0 [2026-10-29 18:14:42,058] INFO : Plotted jets +rank0 [2026-10-29 18:14:43,196] INFO : Plotted jet ratio +rank0 [2026-10-29 18:14:43,277] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:14:46,487] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:14:47,464] INFO : Plotted number of elements +rank0 [2026-10-29 18:14:48,982] INFO : Plotted jets +rank0 [2026-10-29 18:14:50,171] INFO : Plotted jet ratio +rank0 [2026-10-29 18:14:50,252] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits +rank0 [2026-10-29 18:14:52,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits +rank0 [2026-10-29 18:14:53,561] INFO : Plotted number of elements +rank0 [2026-10-29 18:14:55,395] INFO : Plotted jets +rank0 [2026-10-29 18:14:56,573] INFO : Plotted jet ratio +rank0 [2026-10-29 18:18:49,354] INFO : Step 40100/50000 rank0 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0944 | Regression_eta: 0.0473 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0631 | LR: 1.84e-04 | DataLoad Time: 0.0098s | Model Forward Time: 6.0350s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank0 [2026-10-29 18:18:49,595] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 76% +2 6 0x7408, 15961 44.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 47.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 11% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:22:52,058] INFO : Step 40200/50000 rank0 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0821 | Regression_eta: 0.0674 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0804 | LR: 1.83e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.5740s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank0 [2026-10-29 18:22:52,305] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:26:53,767] INFO : Step 40300/50000 rank0 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0823 | Regression_eta: 0.0665 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0911 | LR: 1.81e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2392s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank0 [2026-10-29 18:26:54,021] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:30:44,841] INFO : Step 40400/50000 rank0 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0761 | Regression_eta: 0.0488 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0569 | LR: 1.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6700s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank0 [2026-10-29 18:30:45,095] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:34:32,555] INFO : Step 40500/50000 rank0 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0881 | Regression_eta: 0.0472 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0659 | LR: 1.78e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1629s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank0 [2026-10-29 18:34:32,792] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:38:20,176] INFO : Step 40600/50000 rank0 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0670 | Regression_eta: 0.0624 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0665 | LR: 1.76e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7942s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank0 [2026-10-29 18:38:20,417] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:42:10,282] INFO : Step 40700/50000 rank0 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0892 | Regression_eta: 0.0488 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0631 | LR: 1.75e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1852s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank0 [2026-10-29 18:42:10,520] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 46.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:46:00,508] INFO : Step 40800/50000 rank0 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0851 | Regression_eta: 0.0433 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0576 | LR: 1.73e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.7830s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank0 [2026-10-29 18:46:00,748] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:49:49,840] INFO : Step 40900/50000 rank0 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0707 | Regression_eta: 0.0544 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0730 | LR: 1.72e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3132s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank0 [2026-10-29 18:49:50,078] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:53:42,141] INFO : Step 41000/50000 rank0 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2266 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0966 | Regression_eta: 0.0513 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0733 | LR: 1.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1937s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank0 [2026-10-29 18:53:42,381] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 45% +2 6 0x7408, 15961 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 95% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 4% +4 8 0x7408, 13395 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 46% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:53:42,401] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-41000.pth +rank0 [2026-10-29 18:53:42,584] INFO : Running validation on rank0 +rank0 [2026-10-29 18:53:42,584] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:53:42,584] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION | Step=41000/50000 | Train Loss=1.2889 | Valid Loss=1.2719 | Train Loss Unweighted=0.6706 | Valid Loss Unweighted=0.6412 | Stale=2 | ETA=53.6m +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 18:54:33,875] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari=0.431436 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.573666 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_competing=0.637406 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.590536 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0542912 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0597316 +rank0 [2026-10-29 18:54:33,876] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/seconds_per_rank=22.201 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:54:33,877] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari=0.299624 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding_minus_raw/ari=0.169184 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 18:54:33,878] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:54:33,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:54:34,146] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:54:34,150] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 18:54:59,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:55:00,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:55:01,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:55:03,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:55:13,061] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:55:13,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:55:13,608] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:55:13,612] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:55:29,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:55:30,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:55:32,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:55:33,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:55:43,143] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:55:43,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:55:43,860] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 18:55:43,864] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 18:55:59,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:56:00,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:56:01,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:56:01,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:56:11,784] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 18:56:11,800] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:56:15,012] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:56:16,074] INFO : Plotted number of elements +rank0 [2026-10-29 18:56:17,634] INFO : Plotted jets +rank0 [2026-10-29 18:56:19,096] INFO : Plotted jet ratio +rank0 [2026-10-29 18:56:19,168] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:56:22,338] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:56:23,314] INFO : Plotted number of elements +rank0 [2026-10-29 18:56:24,822] INFO : Plotted jets +rank0 [2026-10-29 18:56:26,021] INFO : Plotted jet ratio +rank0 [2026-10-29 18:56:26,084] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits +rank0 [2026-10-29 18:56:28,532] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits +rank0 [2026-10-29 18:56:29,375] INFO : Plotted number of elements +rank0 [2026-10-29 18:56:30,896] INFO : Plotted jets +rank0 [2026-10-29 18:56:32,395] INFO : Plotted jet ratio +rank0 [2026-10-29 19:00:34,748] INFO : Step 41100/50000 rank0 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0939 | Regression_energy: 0.1026 | Regression_eta: 0.0560 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0828 | LR: 1.69e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7038s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank0 [2026-10-29 19:00:34,989] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:04:28,906] INFO : Step 41200/50000 rank0 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1197 | Regression_eta: 0.0483 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0648 | LR: 1.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6218s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank0 [2026-10-29 19:04:29,160] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:08:18,806] INFO : Step 41300/50000 rank0 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2409 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0908 | Regression_eta: 0.0580 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0785 | LR: 1.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8730s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank0 [2026-10-29 19:08:19,060] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 45.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:12:03,002] INFO : Step 41400/50000 rank0 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0732 | Regression_eta: 0.0470 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0683 | LR: 1.64e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5909s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank0 [2026-10-29 19:12:03,241] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:15:49,350] INFO : Step 41500/50000 rank0 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1513 | Regression_cos_phi: 0.1123 | Regression_energy: 0.0868 | Regression_eta: 0.0498 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0650 | LR: 1.63e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9677s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank0 [2026-10-29 19:15:49,588] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:19:48,805] INFO : Step 41600/50000 rank0 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0847 | Regression_eta: 0.0571 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0678 | LR: 1.61e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4172s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank0 [2026-10-29 19:19:49,056] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:23:42,483] INFO : Step 41700/50000 rank0 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0951 | Regression_eta: 0.0609 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0674 | LR: 1.60e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5874s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank0 [2026-10-29 19:23:42,721] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 45.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 97% +6 10 0x7408, 62036 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:27:39,484] INFO : Step 41800/50000 rank0 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0798 | Regression_eta: 0.0854 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0875 | LR: 1.58e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.9532s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank0 [2026-10-29 19:27:39,724] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 47.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:31:33,096] INFO : Step 41900/50000 rank0 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1708 | Regression_cos_phi: 0.1258 | Regression_energy: 0.0920 | Regression_eta: 0.0670 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0941 | LR: 1.57e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0916s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank0 [2026-10-29 19:31:33,336] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:35:33,630] INFO : Step 42000/50000 rank0 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2270 | Regression_cos_phi: 0.1202 | Regression_energy: 0.0895 | Regression_eta: 0.0746 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0865 | LR: 1.56e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6803s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank0 [2026-10-29 19:35:33,869] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:35:33,891] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-42000.pth +rank0 [2026-10-29 19:35:34,189] INFO : Running validation on rank0 +rank0 [2026-10-29 19:35:34,189] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 19:35:34,189] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION | Step=42000/50000 | Train Loss=1.4172 | Valid Loss=1.3481 | Train Loss Unweighted=0.7107 | Valid Loss Unweighted=0.6916 | Stale=3 | ETA=54.5m +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 19:36:23,328] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari=0.415541 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.546629 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_competing=0.60219 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.582526 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0701868 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0517216 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 19:36:23,329] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/seconds_per_rank=22.2059 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari=0.283692 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 19:36:23,330] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding_minus_raw/ari=0.153253 +rank0 [2026-10-29 19:36:23,331] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 19:36:23,331] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 19:36:23,331] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 19:36:23,337] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:36:23,593] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 19:36:23,605] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 19:36:50,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 19:36:51,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 19:36:52,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 19:36:53,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 19:37:03,825] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 19:37:03,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:37:04,354] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 19:37:04,357] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:37:20,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 19:37:21,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 19:37:22,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 19:37:23,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 19:37:33,516] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 19:37:33,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:37:34,030] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 19:37:34,033] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 19:37:49,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 19:37:50,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 19:37:51,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 19:37:51,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 19:38:01,745] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 19:38:01,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits +rank0 [2026-10-29 19:38:05,053] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits +rank0 [2026-10-29 19:38:06,165] INFO : Plotted number of elements +rank0 [2026-10-29 19:38:07,826] INFO : Plotted jets +rank0 [2026-10-29 19:38:09,355] INFO : Plotted jet ratio +rank0 [2026-10-29 19:38:09,424] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:38:12,674] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:38:13,727] INFO : Plotted number of elements +rank0 [2026-10-29 19:38:15,306] INFO : Plotted jets +rank0 [2026-10-29 19:38:16,544] INFO : Plotted jet ratio +rank0 [2026-10-29 19:38:16,594] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits +rank0 [2026-10-29 19:38:19,147] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits +rank0 [2026-10-29 19:38:20,057] INFO : Plotted number of elements +rank0 [2026-10-29 19:38:21,587] INFO : Plotted jets +rank0 [2026-10-29 19:38:23,100] INFO : Plotted jet ratio +rank0 [2026-10-29 19:42:31,004] INFO : Step 42100/50000 rank0 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0899 | Regression_eta: 0.0517 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0574 | LR: 1.54e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.0029s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank0 [2026-10-29 19:42:31,243] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 5% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 98% +2 6 0x7408, 15961 44.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 24% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 32% 0% +4 8 0x7408, 13395 47.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 32% 0% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 17% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:46:37,494] INFO : Step 42200/50000 rank0 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0914 | Regression_eta: 0.0454 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0900 | LR: 1.53e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5483s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank0 [2026-10-29 19:46:37,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 45.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 98% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +6 10 0x7408, 62036 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 19:50:52,225] INFO : Step 42300/50000 rank0 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2304 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0916 | Regression_eta: 0.0539 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0925 | LR: 1.52e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9131s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank0 [2026-10-29 19:50:52,464] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:54:45,712] INFO : Step 42400/50000 rank0 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0871 | Regression_eta: 0.0534 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0575 | LR: 1.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3465s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank0 [2026-10-29 19:54:45,951] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:58:48,806] INFO : Step 42500/50000 rank0 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2184 | Regression_cos_phi: 0.1106 | Regression_energy: 0.1119 | Regression_eta: 0.0712 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0765 | LR: 1.49e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.8768s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank0 [2026-10-29 19:58:49,056] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:03:00,315] INFO : Step 42600/50000 rank0 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2317 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0904 | Regression_eta: 0.0499 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0894 | LR: 1.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 9.4908s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank0 [2026-10-29 20:03:00,571] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 44.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 46.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +7 11 0x7408, 49174 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:06:58,778] INFO : Step 42700/50000 rank0 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0715 | Regression_energy: 0.1038 | Regression_eta: 0.0665 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0761 | LR: 1.47e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4711s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank0 [2026-10-29 20:06:59,037] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:10:50,460] INFO : Step 42800/50000 rank0 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0925 | Regression_eta: 0.0768 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0836 | LR: 1.45e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3129s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank0 [2026-10-29 20:10:50,701] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:14:51,972] INFO : Step 42900/50000 rank0 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0829 | Regression_eta: 0.0483 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0807 | LR: 1.44e-04 | DataLoad Time: 0.0113s | Model Forward Time: 5.0696s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank0 [2026-10-29 20:14:52,221] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:18:48,373] INFO : Step 43000/50000 rank0 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0893 | Regression_eta: 0.0542 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0709 | LR: 1.43e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.7085s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank0 [2026-10-29 20:18:48,625] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 44.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 47.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:18:48,646] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-43000.pth +rank0 [2026-10-29 20:18:48,985] INFO : Running validation on rank0 +rank0 [2026-10-29 20:18:48,985] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 20:18:48,985] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION | Step=43000/50000 | Train Loss=1.2106 | Valid Loss=1.2653 | Train Loss Unweighted=0.6158 | Valid Loss Unweighted=0.6414 | Stale=4 | ETA=53.6m +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 20:19:39,391] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari=0.418353 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.542429 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_competing=0.59843 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.590889 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.067374 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0600844 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 20:19:39,392] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/seconds_per_rank=22.2101 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:19:39,393] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari=0.298076 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding_minus_raw/ari=0.167636 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 20:19:39,394] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 20:19:39,399] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:19:39,661] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 20:19:39,670] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 20:20:09,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 20:20:10,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 20:20:12,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 20:20:13,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 20:20:23,199] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:20:23,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:20:23,705] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 20:20:23,708] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:20:40,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 20:20:41,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 20:20:42,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 20:20:43,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 20:20:53,362] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:20:53,999] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 20:20:54,003] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 20:21:09,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 20:21:10,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 20:21:11,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 20:21:11,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 20:21:21,941] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 20:21:21,957] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits +rank0 [2026-10-29 20:21:25,186] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits +rank0 [2026-10-29 20:21:26,269] INFO : Plotted number of elements +rank0 [2026-10-29 20:21:27,860] INFO : Plotted jets +rank0 [2026-10-29 20:21:29,007] INFO : Plotted jet ratio +rank0 [2026-10-29 20:21:29,075] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:21:32,324] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:21:33,337] INFO : Plotted number of elements +rank0 [2026-10-29 20:21:35,191] INFO : Plotted jets +rank0 [2026-10-29 20:21:36,374] INFO : Plotted jet ratio +rank0 [2026-10-29 20:21:36,464] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits +rank0 [2026-10-29 20:21:38,771] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits +rank0 [2026-10-29 20:21:39,633] INFO : Plotted number of elements +rank0 [2026-10-29 20:21:41,176] INFO : Plotted jets +rank0 [2026-10-29 20:21:42,346] INFO : Plotted jet ratio +rank0 [2026-10-29 20:25:41,435] INFO : Step 43100/50000 rank0 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0651 | Regression_eta: 0.0359 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0378 | LR: 1.42e-04 | DataLoad Time: 0.0248s | Model Forward Time: 5.1937s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank0 [2026-10-29 20:25:41,684] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 8% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:29:28,844] INFO : Step 43200/50000 rank0 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2278 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0820 | Regression_eta: 0.0719 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0739 | LR: 1.40e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8173s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank0 [2026-10-29 20:29:29,086] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:33:23,888] INFO : Step 43300/50000 rank0 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0898 | Regression_eta: 0.0566 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0585 | LR: 1.39e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0232s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank0 [2026-10-29 20:33:24,127] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:37:10,337] INFO : Step 43400/50000 rank0 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0968 | Regression_eta: 0.0404 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0648 | LR: 1.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1808s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank0 [2026-10-29 20:37:10,576] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:41:00,558] INFO : Step 43500/50000 rank0 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0789 | Regression_eta: 0.0517 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0748 | LR: 1.37e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8128s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank0 [2026-10-29 20:41:00,824] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:44:46,094] INFO : Step 43600/50000 rank0 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2546 | Regression_cos_phi: 0.1116 | Regression_energy: 0.0773 | Regression_eta: 0.0737 | Regression_pt: 0.0809 | Regression_sin_phi: 0.1133 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8538s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank0 [2026-10-29 20:44:46,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:48:26,618] INFO : Step 43700/50000 rank0 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0657 | Regression_eta: 0.0760 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0618 | LR: 1.35e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0833s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank0 [2026-10-29 20:48:26,860] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:52:15,547] INFO : Step 43800/50000 rank0 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0999 | Regression_eta: 0.0405 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0617 | LR: 1.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 7.2807s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank0 [2026-10-29 20:52:15,787] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 44.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:56:19,192] INFO : Step 43900/50000 rank0 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2502 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0757 | Regression_eta: 0.0683 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0888 | LR: 1.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0494s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank0 [2026-10-29 20:56:19,433] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:00:02,468] INFO : Step 44000/50000 rank0 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0994 | Regression_eta: 0.0515 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0755 | LR: 1.32e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9171s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank0 [2026-10-29 21:00:02,708] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:00:02,734] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-44000.pth +rank0 [2026-10-29 21:00:03,284] INFO : Running validation on rank0 +rank0 [2026-10-29 21:00:03,284] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:00:03,284] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION | Step=44000/50000 | Train Loss=1.2618 | Valid Loss=1.2435 | Train Loss Unweighted=0.6263 | Valid Loss Unweighted=0.6281 | Stale=5 | ETA=50.5m +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 21:00:58,031] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari=0.409458 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.539808 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_competing=0.594591 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.598776 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0762694 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0679717 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 21:00:58,032] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/seconds_per_rank=21.9067 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:00:58,033] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari=0.296784 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding_minus_raw/ari=0.166345 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 21:00:58,034] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:00:58,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:00:58,297] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:00:58,308] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 21:01:24,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:01:25,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:01:27,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:01:28,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:01:38,368] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:01:38,954] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:01:38,958] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:01:55,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:01:56,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:01:57,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:01:58,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:02:08,361] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:02:08,957] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 21:02:08,960] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 21:02:24,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:02:25,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:02:26,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:02:26,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:02:36,911] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 21:02:36,927] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:02:40,214] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:02:41,306] INFO : Plotted number of elements +rank0 [2026-10-29 21:02:42,818] INFO : Plotted jets +rank0 [2026-10-29 21:02:43,944] INFO : Plotted jet ratio +rank0 [2026-10-29 21:02:44,008] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:02:47,210] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:02:48,632] INFO : Plotted number of elements +rank0 [2026-10-29 21:02:50,270] INFO : Plotted jets +rank0 [2026-10-29 21:02:51,474] INFO : Plotted jet ratio +rank0 [2026-10-29 21:02:51,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits +rank0 [2026-10-29 21:02:53,965] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits +rank0 [2026-10-29 21:02:54,822] INFO : Plotted number of elements +rank0 [2026-10-29 21:02:56,335] INFO : Plotted jets +rank0 [2026-10-29 21:02:57,498] INFO : Plotted jet ratio +rank0 [2026-10-29 21:06:43,363] INFO : Step 44100/50000 rank0 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0861 | Regression_eta: 0.0449 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0714 | LR: 1.31e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8485s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank0 [2026-10-29 21:06:43,603] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 44.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:10:41,492] INFO : Step 44200/50000 rank0 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0723 | Regression_eta: 0.0437 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0655 | LR: 1.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.7262s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank0 [2026-10-29 21:10:41,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 45.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 47.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 6% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 44% +6 10 0x7408, 62036 52.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:14:30,181] INFO : Step 44300/50000 rank0 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2204 | Regression_cos_phi: 0.1187 | Regression_energy: 0.0917 | Regression_eta: 0.0502 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0722 | LR: 1.29e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7626s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank0 [2026-10-29 21:14:30,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:18:21,106] INFO : Step 44400/50000 rank0 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0831 | Regression_eta: 0.0517 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0701 | LR: 1.28e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5313s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank0 [2026-10-29 21:18:21,345] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:22:17,083] INFO : Step 44500/50000 rank0 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0910 | Regression_eta: 0.0696 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0776 | LR: 1.27e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5267s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank0 [2026-10-29 21:22:17,325] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:26:11,320] INFO : Step 44600/50000 rank0 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0844 | Regression_eta: 0.0531 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0801 | LR: 1.26e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8736s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank0 [2026-10-29 21:26:11,557] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:30:16,761] INFO : Step 44700/50000 rank0 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0675 | Regression_eta: 0.0399 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0629 | LR: 1.25e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3625s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank0 [2026-10-29 21:30:17,012] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 44.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:34:10,710] INFO : Step 44800/50000 rank0 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0727 | Regression_eta: 0.0459 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0359 | LR: 1.24e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1665s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank0 [2026-10-29 21:34:10,961] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:38:11,681] INFO : Step 44900/50000 rank0 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2210 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0958 | Regression_eta: 0.0531 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0578 | LR: 1.23e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6278s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank0 [2026-10-29 21:38:11,922] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 44.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:42:14,312] INFO : Step 45000/50000 rank0 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2194 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0970 | Regression_eta: 0.0809 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0760 | LR: 1.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6844s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank0 [2026-10-29 21:42:14,556] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 6% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 23% +2 6 0x7408, 15961 45.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:42:14,606] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-45000.pth +rank0 [2026-10-29 21:42:15,086] INFO : Running validation on rank0 +rank0 [2026-10-29 21:42:15,086] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:42:15,087] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION | Step=45000/50000 | Train Loss=1.3238 | Valid Loss=1.2398 | Train Loss Unweighted=0.6669 | Valid Loss Unweighted=0.6227 | Stale=6 | ETA=45.9m +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:43:07,009] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari=0.407222 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.545115 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_competing=0.600709 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.609302 +rank0 [2026-10-29 21:43:07,010] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0785053 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0784973 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/seconds_per_rank=22.0859 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 21:43:07,011] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari=0.301752 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding_minus_raw/ari=0.171312 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 21:43:07,012] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:43:07,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:43:07,272] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:43:07,282] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 21:43:32,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:43:33,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:43:34,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:43:37,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:43:47,113] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 21:43:47,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:43:47,728] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:43:47,732] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:44:04,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:44:05,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:44:06,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:44:07,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:44:17,159] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 21:44:17,170] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:44:17,714] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 21:44:17,717] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 21:44:33,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:44:34,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:44:34,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:44:35,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:44:45,606] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 21:44:45,919] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:44:49,243] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:44:50,330] INFO : Plotted number of elements +rank0 [2026-10-29 21:44:51,894] INFO : Plotted jets +rank0 [2026-10-29 21:44:53,047] INFO : Plotted jet ratio +rank0 [2026-10-29 21:44:53,130] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:44:56,306] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:44:57,642] INFO : Plotted number of elements +rank0 [2026-10-29 21:44:59,161] INFO : Plotted jets +rank0 [2026-10-29 21:45:00,354] INFO : Plotted jet ratio +rank0 [2026-10-29 21:45:00,407] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits +rank0 [2026-10-29 21:45:02,701] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits +rank0 [2026-10-29 21:45:03,544] INFO : Plotted number of elements +rank0 [2026-10-29 21:45:05,064] INFO : Plotted jets +rank0 [2026-10-29 21:45:06,238] INFO : Plotted jet ratio +rank0 [2026-10-29 21:48:56,304] INFO : Step 45100/50000 rank0 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0791 | Regression_eta: 0.0410 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0625 | LR: 1.21e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0959s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank0 [2026-10-29 21:48:56,545] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:52:46,210] INFO : Step 45200/50000 rank0 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0931 | Regression_eta: 0.0427 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0697 | LR: 1.20e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9745s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank0 [2026-10-29 21:52:46,450] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:56:36,270] INFO : Step 45300/50000 rank0 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0760 | Regression_eta: 0.0443 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0595 | LR: 1.19e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6595s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank0 [2026-10-29 21:56:36,511] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:00:33,640] INFO : Step 45400/50000 rank0 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0805 | Regression_eta: 0.0523 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0900 | LR: 1.19e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6153s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank0 [2026-10-29 22:00:33,879] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:04:23,917] INFO : Step 45500/50000 rank0 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0769 | Regression_eta: 0.0540 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0545 | LR: 1.18e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3196s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank0 [2026-10-29 22:04:24,160] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:08:21,957] INFO : Step 45600/50000 rank0 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2468 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0799 | Regression_eta: 0.0744 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0875 | LR: 1.17e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7786s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank0 [2026-10-29 22:08:22,197] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:12:12,018] INFO : Step 45700/50000 rank0 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0749 | Regression_eta: 0.0485 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0775 | LR: 1.16e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7305s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank0 [2026-10-29 22:12:12,258] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:16:09,343] INFO : Step 45800/50000 rank0 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0937 | Regression_energy: 0.0846 | Regression_eta: 0.0631 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0710 | LR: 1.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4069s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank0 [2026-10-29 22:16:09,586] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 44.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 47.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:20:19,359] INFO : Step 45900/50000 rank0 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0855 | Regression_eta: 0.0601 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0753 | LR: 1.15e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6094s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank0 [2026-10-29 22:20:19,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:24:25,162] INFO : Step 46000/50000 rank0 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0684 | Regression_eta: 0.0419 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0583 | LR: 1.14e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.0088s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank0 [2026-10-29 22:24:25,401] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 44.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:24:25,422] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-46000.pth +rank0 [2026-10-29 22:24:25,607] INFO : Running validation on rank0 +rank0 [2026-10-29 22:24:25,607] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 22:24:25,607] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION | Step=46000/50000 | Train Loss=1.2145 | Valid Loss=1.2179 | Train Loss Unweighted=0.6133 | Valid Loss Unweighted=0.6074 | Stale=7 | ETA=39.6m +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 22:25:17,209] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari=0.40281 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.572594 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_competing=0.634922 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.595933 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0829174 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0651278 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 22:25:17,210] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/seconds_per_rank=22.1352 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:25:17,211] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari=0.300092 +rank0 [2026-10-29 22:25:17,212] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 22:25:17,212] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding_minus_raw/ari=0.169652 +rank0 [2026-10-29 22:25:17,212] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 22:25:17,212] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 22:25:17,212] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 22:25:17,216] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:25:17,475] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 22:25:17,478] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 22:25:46,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 22:25:47,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 22:25:48,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 22:25:49,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 22:25:59,975] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 22:25:59,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:26:00,493] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 22:26:00,497] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:26:16,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 22:26:17,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 22:26:20,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 22:26:21,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 22:26:31,416] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:26:31,920] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 22:26:31,924] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 22:26:47,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 22:26:48,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 22:26:49,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 22:26:49,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 22:26:59,984] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 22:27:00,038] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits +rank0 [2026-10-29 22:27:03,349] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits +rank0 [2026-10-29 22:27:04,444] INFO : Plotted number of elements +rank0 [2026-10-29 22:27:05,925] INFO : Plotted jets +rank0 [2026-10-29 22:27:07,067] INFO : Plotted jet ratio +rank0 [2026-10-29 22:27:07,161] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:27:10,364] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:27:11,710] INFO : Plotted number of elements +rank0 [2026-10-29 22:27:13,228] INFO : Plotted jets +rank0 [2026-10-29 22:27:14,406] INFO : Plotted jet ratio +rank0 [2026-10-29 22:27:14,488] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits +rank0 [2026-10-29 22:27:16,808] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits +rank0 [2026-10-29 22:27:17,665] INFO : Plotted number of elements +rank0 [2026-10-29 22:27:19,191] INFO : Plotted jets +rank0 [2026-10-29 22:27:20,359] INFO : Plotted jet ratio +rank0 [2026-10-29 22:31:17,291] INFO : Step 46100/50000 rank0 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0735 | Regression_eta: 0.0520 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0730 | LR: 1.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7293s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank0 [2026-10-29 22:31:17,535] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:35:13,707] INFO : Step 46200/50000 rank0 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0789 | Regression_eta: 0.0430 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0590 | LR: 1.13e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8271s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank0 [2026-10-29 22:35:13,948] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:39:25,801] INFO : Step 46300/50000 rank0 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0865 | Regression_eta: 0.0310 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0434 | LR: 1.12e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6362s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank0 [2026-10-29 22:39:26,042] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:43:24,535] INFO : Step 46400/50000 rank0 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0739 | Regression_eta: 0.0465 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0743 | LR: 1.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.4173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank0 [2026-10-29 22:43:24,781] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:47:18,050] INFO : Step 46500/50000 rank0 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0897 | Regression_eta: 0.0539 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0728 | LR: 1.11e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.2720s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank0 [2026-10-29 22:47:18,289] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 3% +4 8 0x7408, 13395 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 89% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:51:21,408] INFO : Step 46600/50000 rank0 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2059 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0748 | Regression_eta: 0.0593 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0566 | LR: 1.10e-04 | DataLoad Time: 0.0109s | Model Forward Time: 6.2377s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank0 [2026-10-29 22:51:21,655] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 94% +2 6 0x7408, 15961 44.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 47.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 94% +6 10 0x7408, 62036 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:55:26,893] INFO : Step 46700/50000 rank0 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0692 | Regression_eta: 0.0515 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0574 | LR: 1.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7258s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank0 [2026-10-29 22:55:27,137] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:59:27,493] INFO : Step 46800/50000 rank0 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0772 | Regression_eta: 0.0344 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0699 | LR: 1.09e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.5979s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank0 [2026-10-29 22:59:27,735] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 44.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 56% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 47.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 52.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 94% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:04:16,182] INFO : Step 46900/50000 rank0 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0793 | Regression_eta: 0.0460 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0639 | LR: 1.09e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6535s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank0 [2026-10-29 23:04:16,420] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:08:16,179] INFO : Step 47000/50000 rank0 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0809 | Regression_eta: 0.0688 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0715 | LR: 1.08e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.0099s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank0 [2026-10-29 23:08:16,420] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 81% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:08:16,526] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-47000.pth +rank0 [2026-10-29 23:08:16,709] INFO : Running validation on rank0 +rank0 [2026-10-29 23:08:16,709] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 23:08:16,709] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION | Step=47000/50000 | Train Loss=1.2560 | Valid Loss=1.1979 | Train Loss Unweighted=0.6143 | Valid Loss Unweighted=0.5973 | Stale=0 | ETA=31.8m +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 23:09:08,290] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari=0.398812 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.569597 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_competing=0.633056 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.589933 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0869157 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0591281 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 23:09:08,291] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/seconds_per_rank=22.1459 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 23:09:08,292] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari=0.285876 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding_minus_raw/ari=0.155436 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 23:09:08,293] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 23:09:08,298] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:09:08,570] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 23:09:08,580] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 23:09:36,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 23:09:42,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 23:09:44,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 23:09:45,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 23:09:55,108] INFO : Time taken to make predictions on device 0 is: 0.36 min +rank0 [2026-10-29 23:09:55,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:09:55,737] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 23:09:55,741] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:10:12,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 23:10:13,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 23:10:14,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 23:10:15,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 23:10:25,216] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:10:25,935] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 23:10:25,938] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 23:10:41,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 23:10:42,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 23:10:43,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 23:10:44,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 23:10:54,085] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 23:10:54,104] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:10:57,551] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:10:58,673] INFO : Plotted number of elements +rank0 [2026-10-29 23:11:00,245] INFO : Plotted jets +rank0 [2026-10-29 23:11:01,464] INFO : Plotted jet ratio +rank0 [2026-10-29 23:11:01,588] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:11:04,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:11:05,867] INFO : Plotted number of elements +rank0 [2026-10-29 23:11:07,715] INFO : Plotted jets +rank0 [2026-10-29 23:11:08,834] INFO : Plotted jet ratio +rank0 [2026-10-29 23:11:08,898] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits +rank0 [2026-10-29 23:11:11,202] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits +rank0 [2026-10-29 23:11:12,051] INFO : Plotted number of elements +rank0 [2026-10-29 23:11:13,534] INFO : Plotted jets +rank0 [2026-10-29 23:11:14,708] INFO : Plotted jet ratio +rank0 [2026-10-29 23:15:14,665] INFO : Step 47100/50000 rank0 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0849 | Regression_eta: 0.0452 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0508 | LR: 1.07e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4263s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank0 [2026-10-29 23:15:14,909] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 44.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:19:20,766] INFO : Step 47200/50000 rank0 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0733 | Regression_eta: 0.0396 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0508 | LR: 1.07e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.6704s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank0 [2026-10-29 23:19:21,009] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 44.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 47.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:23:32,147] INFO : Step 47300/50000 rank0 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0803 | Regression_eta: 0.0460 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0544 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3689s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank0 [2026-10-29 23:23:32,395] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 45.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:27:30,313] INFO : Step 47400/50000 rank0 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0303 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0848 | Regression_eta: 0.0481 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0644 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7300s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank0 [2026-10-29 23:27:30,554] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:31:39,248] INFO : Step 47500/50000 rank0 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0648 | Regression_eta: 0.0578 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0816 | LR: 1.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5023s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank0 [2026-10-29 23:31:39,492] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 44.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:35:35,019] INFO : Step 47600/50000 rank0 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2447 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0791 | Regression_eta: 0.0695 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0919 | LR: 1.05e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.1662s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank0 [2026-10-29 23:35:35,260] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:39:49,965] INFO : Step 47700/50000 rank0 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0920 | Regression_eta: 0.0658 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0739 | LR: 1.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.5984s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank0 [2026-10-29 23:39:50,209] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:43:54,702] INFO : Step 47800/50000 rank0 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0795 | Regression_eta: 0.0337 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0564 | LR: 1.04e-04 | DataLoad Time: 0.0112s | Model Forward Time: 5.3612s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank0 [2026-10-29 23:43:54,952] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 8% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 44.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 89% +4 8 0x7408, 13395 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:47:49,466] INFO : Step 47900/50000 rank0 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0844 | Regression_eta: 0.0548 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0861 | LR: 1.04e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8669s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank0 [2026-10-29 23:47:49,708] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:51:39,394] INFO : Step 48000/50000 rank0 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2277 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0806 | Regression_eta: 0.0663 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0761 | LR: 1.04e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9611s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank0 [2026-10-29 23:51:39,632] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:51:39,749] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-48000.pth +rank0 [2026-10-29 23:51:40,033] INFO : Running validation on rank0 +rank0 [2026-10-29 23:51:40,033] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 23:51:40,033] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 23:52:31,756] INFO : VALIDATION | Step=48000/50000 | Train Loss=1.2878 | Valid Loss=1.2037 | Train Loss Unweighted=0.6389 | Valid Loss Unweighted=0.5982 | Stale=1 | ETA=22.6m +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 23:52:31,757] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari=0.437181 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.556457 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_competing=0.615482 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.595083 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0485465 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0642781 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 23:52:31,758] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/seconds_per_rank=22.0496 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari=0.30522 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding_minus_raw/ari=0.17478 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 23:52:31,759] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 23:52:31,760] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 23:52:31,765] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:52:32,018] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 23:52:32,037] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 23:52:58,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 23:53:00,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 23:53:02,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 23:53:03,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 23:53:13,256] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 23:53:13,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:53:13,761] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 23:53:13,765] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:53:30,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 23:53:31,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 23:53:32,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 23:53:33,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 23:53:43,343] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 23:53:43,354] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:53:43,871] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 23:53:43,875] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 23:53:59,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 23:53:59,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 23:54:00,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 23:54:01,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 23:54:11,462] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 23:54:11,674] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:54:15,037] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:54:16,123] INFO : Plotted number of elements +rank0 [2026-10-29 23:54:17,748] INFO : Plotted jets +rank0 [2026-10-29 23:54:18,974] INFO : Plotted jet ratio +rank0 [2026-10-29 23:54:19,041] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:54:22,383] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:54:23,391] INFO : Plotted number of elements +rank0 [2026-10-29 23:54:25,314] INFO : Plotted jets +rank0 [2026-10-29 23:54:26,503] INFO : Plotted jet ratio +rank0 [2026-10-29 23:54:26,557] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits +rank0 [2026-10-29 23:54:28,898] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits +rank0 [2026-10-29 23:54:29,748] INFO : Plotted number of elements +rank0 [2026-10-29 23:54:31,279] INFO : Plotted jets +rank0 [2026-10-29 23:54:32,460] INFO : Plotted jet ratio +rank0 [2026-10-30 00:00:54,249] INFO : Step 48100/50000 rank0 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0789 | Regression_eta: 0.0627 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0853 | LR: 1.03e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2451s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank0 [2026-10-30 00:00:54,491] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +2 6 0x7408, 15961 44.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +7 11 0x7408, 49174 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:04:41,398] INFO : Step 48200/50000 rank0 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0765 | Regression_eta: 0.0564 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0656 | LR: 1.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8433s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank0 [2026-10-30 00:04:41,636] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 44.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:08:32,485] INFO : Step 48300/50000 rank0 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0857 | Regression_eta: 0.0555 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0892 | LR: 1.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.6996s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank0 [2026-10-30 00:08:32,741] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +2 6 0x7408, 15961 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:12:20,486] INFO : Step 48400/50000 rank0 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0866 | Regression_eta: 0.0564 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0669 | LR: 1.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1502s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank0 [2026-10-30 00:12:20,739] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 52% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:16:01,955] INFO : Step 48500/50000 rank0 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0896 | Regression_eta: 0.0495 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0522 | LR: 1.02e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3556s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank0 [2026-10-30 00:16:02,195] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:19:51,155] INFO : Step 48600/50000 rank0 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0705 | Regression_eta: 0.0568 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0945 | LR: 1.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6427s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank0 [2026-10-30 00:19:51,396] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 45.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:23:39,244] INFO : Step 48700/50000 rank0 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0913 | Regression_eta: 0.0621 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0802 | LR: 1.02e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7808s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank0 [2026-10-30 00:23:39,481] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:27:31,086] INFO : Step 48800/50000 rank0 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0274 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0863 | Regression_eta: 0.0371 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0561 | LR: 1.01e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1667s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank0 [2026-10-30 00:27:31,325] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 47% +2 6 0x7408, 15961 45.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 1% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 21% +4 8 0x7408, 13395 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +6 10 0x7408, 62036 52.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:31:25,119] INFO : Step 48900/50000 rank0 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0740 | Regression_eta: 0.0400 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0679 | LR: 1.01e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.3692s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank0 [2026-10-30 00:31:25,360] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:35:14,793] INFO : Step 49000/50000 rank0 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2287 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0879 | Regression_eta: 0.0633 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0712 | LR: 1.01e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.2473s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank0 [2026-10-30 00:35:15,045] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 45.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:35:15,067] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-49000.pth +rank0 [2026-10-30 00:35:15,251] INFO : Running validation on rank0 +rank0 [2026-10-30 00:35:15,251] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:35:15,251] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:36:04,121] INFO : VALIDATION | Step=49000/50000 | Train Loss=1.2892 | Valid Loss=1.1996 | Train Loss Unweighted=0.6428 | Valid Loss Unweighted=0.5974 | Stale=2 | ETA=12.0m +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 00:36:04,122] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari=0.378826 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.561439 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_competing=0.621813 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.6001 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.106902 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0692952 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 00:36:04,123] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/seconds_per_rank=22.1851 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari=0.29557 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 00:36:04,124] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding_minus_raw/ari=0.16513 +rank0 [2026-10-30 00:36:04,125] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 00:36:04,125] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 00:36:04,125] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:36:04,130] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:36:04,387] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:36:04,392] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 00:36:30,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:36:32,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:36:35,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:36:38,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:36:48,224] INFO : Time taken to make predictions on device 0 is: 0.34 min +rank0 [2026-10-30 00:36:48,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:36:48,746] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:36:48,749] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:37:04,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:37:05,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:37:06,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:37:07,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:37:17,477] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:37:18,488] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 00:37:18,491] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 00:37:33,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 00:37:34,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 00:37:35,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 00:37:35,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 00:37:46,000] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-30 00:37:46,039] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits +rank0 [2026-10-30 00:37:49,336] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits +rank0 [2026-10-30 00:37:50,358] INFO : Plotted number of elements +rank0 [2026-10-30 00:37:51,920] INFO : Plotted jets +rank0 [2026-10-30 00:37:53,093] INFO : Plotted jet ratio +rank0 [2026-10-30 00:37:53,156] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:37:56,507] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:37:57,863] INFO : Plotted number of elements +rank0 [2026-10-30 00:37:59,412] INFO : Plotted jets +rank0 [2026-10-30 00:38:00,551] INFO : Plotted jet ratio +rank0 [2026-10-30 00:38:00,631] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits +rank0 [2026-10-30 00:38:02,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits +rank0 [2026-10-30 00:38:03,771] INFO : Plotted number of elements +rank0 [2026-10-30 00:38:05,285] INFO : Plotted jets +rank0 [2026-10-30 00:38:06,443] INFO : Plotted jet ratio +rank0 [2026-10-30 00:41:50,576] INFO : Step 49100/50000 rank0 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0757 | Regression_eta: 0.0392 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0573 | LR: 1.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2626s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank0 [2026-10-30 00:41:50,815] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 27% 100% +2 6 0x7408, 15961 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 48.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:45:59,913] INFO : Step 49200/50000 rank0 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0748 | Regression_eta: 0.0339 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0626 | LR: 1.01e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank0 [2026-10-30 00:46:00,154] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 44.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:49:55,140] INFO : Step 49300/50000 rank0 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0834 | Regression_eta: 0.0487 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0850 | LR: 1.00e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.7041s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank0 [2026-10-30 00:49:55,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 45.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 66% +4 8 0x7408, 13395 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:53:44,914] INFO : Step 49400/50000 rank0 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0742 | Regression_eta: 0.0574 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0799 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6209s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank0 [2026-10-30 00:53:45,154] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:57:40,089] INFO : Step 49500/50000 rank0 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0705 | Regression_eta: 0.0443 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0572 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6342s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank0 [2026-10-30 00:57:40,341] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 44.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:01:39,807] INFO : Step 49600/50000 rank0 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0785 | Regression_eta: 0.0601 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0861 | LR: 1.00e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.4695s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank0 [2026-10-30 01:01:40,047] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 94% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:05:31,580] INFO : Step 49700/50000 rank0 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0873 | Regression_eta: 0.0536 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0698 | LR: 1.00e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.1960s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank0 [2026-10-30 01:05:31,823] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:09:22,630] INFO : Step 49800/50000 rank0 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0247 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0976 | Regression_eta: 0.0832 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0634 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0553s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank0 [2026-10-30 01:09:22,888] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 45.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:13:14,229] INFO : Step 49900/50000 rank0 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0806 | Regression_eta: 0.0435 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0538 | LR: 1.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8182s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank0 [2026-10-30 01:13:14,470] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 44.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:17:09,300] INFO : Step 50000/50000 rank0 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2528 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0888 | Regression_eta: 0.0690 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0919 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.4841s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank0 [2026-10-30 01:17:09,537] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:17:09,640] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-50000.pth +rank0 [2026-10-30 01:17:09,824] INFO : Running validation on rank0 +rank0 [2026-10-30 01:17:09,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 01:17:09,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION | Step=50000/50000 | Train Loss=1.3382 | Valid Loss=1.2301 | Train Loss Unweighted=0.6395 | Valid Loss Unweighted=0.6090 | Stale=3 | ETA=0.0m +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 01:18:03,278] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari=0.440286 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.54449 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_competing=0.600835 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.589046 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0454409 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0582408 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 01:18:03,279] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/seconds_per_rank=21.8835 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 01:18:03,280] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari=0.287626 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding_minus_raw/ari=0.157187 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 01:18:03,281] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 01:18:03,286] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:18:03,533] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 01:18:03,550] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 01:18:29,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 01:18:30,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 01:18:31,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 01:18:32,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 01:18:42,577] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:18:42,590] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:18:43,375] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 01:18:43,378] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:18:59,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 01:19:00,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 01:19:01,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 01:19:02,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 01:19:12,803] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:19:13,324] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 01:19:13,327] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 01:19:28,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:19:29,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:19:30,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:19:31,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:19:41,268] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-30 01:19:41,284] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:19:44,533] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:19:45,608] INFO : Plotted number of elements +rank0 [2026-10-30 01:19:47,161] INFO : Plotted jets +rank0 [2026-10-30 01:19:48,289] INFO : Plotted jet ratio +rank0 [2026-10-30 01:19:48,375] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:19:51,669] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:19:52,642] INFO : Plotted number of elements +rank0 [2026-10-30 01:19:54,133] INFO : Plotted jets +rank0 [2026-10-30 01:19:55,704] INFO : Plotted jet ratio +rank0 [2026-10-30 01:19:55,758] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits +rank0 [2026-10-30 01:19:58,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits +rank0 [2026-10-30 01:19:58,970] INFO : Plotted number of elements +rank0 [2026-10-30 01:20:00,487] INFO : Plotted jets +rank0 [2026-10-30 01:20:01,650] INFO : Plotted jet ratio +rank0 [2026-10-30 01:20:02,085] INFO : Training completed. Total time on device 0: 629.700min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.1 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..1e109fae9a127dbad795c64510a1cbd159e64806 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.1 @@ -0,0 +1,2483 @@ +rank1 [2026-10-28 11:12:32,926] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-28 11:12:32,927] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank1 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-28 11:12:57,452] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-28 11:12:57,452] INFO : nn0 parameters: 171776 +rank1 [2026-10-28 11:12:57,452] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank1 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,456] INFO : Initializing backbone layer 1 +rank1 [2026-10-28 11:12:57,456] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 2 +rank1 [2026-10-28 11:12:57,460] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,463] INFO : Initializing backbone layer 3 +rank1 [2026-10-28 11:12:57,463] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,466] INFO : Initializing backbone layer 4 +rank1 [2026-10-28 11:12:57,466] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,469] INFO : Initializing backbone layer 5 +rank1 [2026-10-28 11:12:57,469] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:57,472] INFO : Backbone initialization took 0.02s +rank1 [2026-10-28 11:12:57,473] INFO : backbone parameters: 2374656 +rank1 [2026-10-28 11:12:57,473] INFO : Initializing output DNNs +rank1 [2026-10-28 11:12:57,522] INFO : Output DNNs initialization took 0.05s +rank1 [2026-10-28 11:12:57,523] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-28 11:12:57,523] INFO : jagged_attention enabled=True +rank1 [2026-10-28 11:12:57,523] INFO : flash_attn_varlen enabled=True +rank1 [2026-10-28 11:12:57,523] INFO : task_queries enabled=False +rank1 [2026-10-28 11:12:57,523] INFO : classification_norm parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : regression_norm parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : classification_query parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank1 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank1 [2026-10-28 11:12:57,523] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-28 11:12:57,524] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_pid parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_pt parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_eta parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : nn_energy parameters: 0 +rank1 [2026-10-28 11:12:57,524] INFO : Total MLPF parameters: 7260244 +rank1 [2026-10-28 11:12:57,524] INFO : MLPF __init__ done +rank1 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank1 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=1 +rank1 [2026-10-28 11:12:57,558] INFO : Moved model to device rank=1 +rank1 [2026-10-28 11:12:57,560] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-28 11:12:57,560] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-28 11:12:57,813] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank1 [2026-10-28 11:12:57,813] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-28 11:12:57,814] INFO : Trainable parameters: 7260244 +rank1 [2026-10-28 11:12:57,814] INFO : Non-trainable parameters: 0 +rank1 [2026-10-28 11:12:57,814] INFO : Total parameters: 7260244 +rank1 [2026-10-28 11:12:57,817] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-28 11:12:57,817] INFO : Getting dataloaders +rank1 [2026-10-28 11:12:57,817] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:03,703] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:04,768] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-28 11:13:06,004] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-28 11:13:06,004] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,280] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,554] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,814] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-28 11:13:06,814] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-28 11:13:21,077] INFO : Got dataloaders +rank1 [2026-10-28 11:13:21,077] INFO : loader split=train rank=1 len=21108 +rank1 [2026-10-28 11:13:21,077] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-28 11:13:21,077] INFO : Starting training from step 1 to 50000 on rank 1 of 8 +rank1 [2026-10-28 11:13:21,077] INFO : Creating train_iterator +rank1 [2026-10-28 11:13:21,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-28 11:13:35,606] INFO : Created train_iterator +rank1 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank1 [2026-10-28 11:17:20,491] INFO : Step 100/50000 rank1 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8246 | Classification_binary: 0.4381 | Regression_cos_phi: 0.0070 | Regression_energy: 1.0853 | Regression_eta: 0.0211 | Regression_pt: 1.0020 | Regression_sin_phi: 0.0086 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.1332s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank1 [2026-10-28 11:20:57,690] INFO : Step 200/50000 rank1 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4383 | Classification_binary: 0.5325 | Regression_cos_phi: 0.3776 | Regression_energy: 0.3283 | Regression_eta: 0.3386 | Regression_pt: 0.3602 | Regression_sin_phi: 0.4235 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 3.5596s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank1 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank1 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3385 | Classification_binary: 0.4386 | Regression_cos_phi: 0.4034 | Regression_energy: 0.3612 | Regression_eta: 0.4691 | Regression_pt: 0.3734 | Regression_sin_phi: 0.3592 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.7565s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank1 [2026-10-28 11:28:09,552] INFO : Step 400/50000 rank1 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2822 | Classification_binary: 0.4238 | Regression_cos_phi: 0.3044 | Regression_energy: 0.3644 | Regression_eta: 0.2968 | Regression_pt: 0.3794 | Regression_sin_phi: 0.2797 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.5822s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank1 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank1 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3571 | Classification_binary: 0.5265 | Regression_cos_phi: 0.3130 | Regression_energy: 0.2833 | Regression_eta: 0.3250 | Regression_pt: 0.2994 | Regression_sin_phi: 0.3503 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 4.0258s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank1 [2026-10-28 11:35:32,722] INFO : Step 600/50000 rank1 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2342 | Classification_binary: 0.5041 | Regression_cos_phi: 0.2942 | Regression_energy: 0.2799 | Regression_eta: 0.2427 | Regression_pt: 0.3100 | Regression_sin_phi: 0.2173 | LR: 1.00e-03 | DataLoad Time: 0.0139s | Model Forward Time: 4.1168s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank1 [2026-10-28 11:39:09,491] INFO : Step 700/50000 rank1 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2298 | Classification_binary: 0.4224 | Regression_cos_phi: 0.2207 | Regression_energy: 0.3049 | Regression_eta: 0.2283 | Regression_pt: 0.3164 | Regression_sin_phi: 0.2672 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 3.3984s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank1 [2026-10-28 11:42:49,057] INFO : Step 800/50000 rank1 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2859 | Classification_binary: 0.4452 | Regression_cos_phi: 0.2368 | Regression_energy: 0.2824 | Regression_eta: 0.3065 | Regression_pt: 0.2933 | Regression_sin_phi: 0.2524 | LR: 9.99e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.6503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank1 [2026-10-28 11:46:31,920] INFO : Step 900/50000 rank1 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2587 | Classification_binary: 0.5089 | Regression_cos_phi: 0.3245 | Regression_energy: 0.3138 | Regression_eta: 0.2393 | Regression_pt: 0.3309 | Regression_sin_phi: 0.2215 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5675s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank1 [2026-10-28 11:50:09,949] INFO : Step 1000/50000 rank1 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2238 | Classification_binary: 0.4745 | Regression_cos_phi: 0.2650 | Regression_energy: 0.2464 | Regression_eta: 0.1844 | Regression_pt: 0.2591 | Regression_sin_phi: 0.2125 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.4180s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank1 [2026-10-28 11:50:10,910] INFO : Running validation on rank1 +rank1 [2026-10-28 11:50:10,910] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 11:50:10,910] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 11:50:58,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:50:59,089] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 11:51:19,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 11:51:20,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 11:51:21,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 11:51:22,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 11:51:32,382] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 11:51:32,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:51:33,313] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 11:51:49,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 11:51:50,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 11:51:51,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 11:51:52,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 11:52:02,149] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 11:52:02,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:52:03,469] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 11:52:18,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 11:52:19,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 11:52:20,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 11:52:20,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 11:52:30,810] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 11:56:43,806] INFO : Step 1100/50000 rank1 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1827 | Classification_binary: 0.3536 | Regression_cos_phi: 0.2798 | Regression_energy: 0.2861 | Regression_eta: 0.2918 | Regression_pt: 0.2895 | Regression_sin_phi: 0.3565 | LR: 9.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9232s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank1 [2026-10-28 12:00:30,891] INFO : Step 1200/50000 rank1 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2730 | Classification_binary: 0.5049 | Regression_cos_phi: 0.2721 | Regression_energy: 0.3050 | Regression_eta: 0.2396 | Regression_pt: 0.3161 | Regression_sin_phi: 0.3256 | LR: 9.99e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4000s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank1 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank1 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2339 | Classification_binary: 0.4145 | Regression_cos_phi: 0.2608 | Regression_energy: 0.2966 | Regression_eta: 0.2593 | Regression_pt: 0.3105 | Regression_sin_phi: 0.2193 | LR: 9.98e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1258s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank1 [2026-10-28 12:08:10,851] INFO : Step 1400/50000 rank1 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2108 | Classification_binary: 0.3681 | Regression_cos_phi: 0.4172 | Regression_energy: 0.2684 | Regression_eta: 0.3311 | Regression_pt: 0.2735 | Regression_sin_phi: 0.3207 | LR: 9.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7540s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank1 [2026-10-28 12:11:57,646] INFO : Step 1500/50000 rank1 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2250 | Classification_binary: 0.4304 | Regression_cos_phi: 0.2906 | Regression_energy: 0.2313 | Regression_eta: 0.2472 | Regression_pt: 0.2405 | Regression_sin_phi: 0.3150 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5797s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank1 [2026-10-28 12:15:43,592] INFO : Step 1600/50000 rank1 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2432 | Classification_binary: 0.4946 | Regression_cos_phi: 0.3185 | Regression_energy: 0.2975 | Regression_eta: 0.2366 | Regression_pt: 0.3158 | Regression_sin_phi: 0.3451 | LR: 9.98e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.0373s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank1 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank1 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2080 | Classification_binary: 0.3865 | Regression_cos_phi: 0.1995 | Regression_energy: 0.2466 | Regression_eta: 0.2473 | Regression_pt: 0.2585 | Regression_sin_phi: 0.2469 | LR: 9.97e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.3915s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank1 [2026-10-28 12:23:17,460] INFO : Step 1800/50000 rank1 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2046 | Classification_binary: 0.4002 | Regression_cos_phi: 0.1841 | Regression_energy: 0.2285 | Regression_eta: 0.1529 | Regression_pt: 0.2438 | Regression_sin_phi: 0.2768 | LR: 9.97e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9412s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank1 [2026-10-28 12:27:10,965] INFO : Step 1900/50000 rank1 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1874 | Classification_binary: 0.4245 | Regression_cos_phi: 0.1797 | Regression_energy: 0.3035 | Regression_eta: 0.1554 | Regression_pt: 0.3151 | Regression_sin_phi: 0.1873 | LR: 9.97e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.5691s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank1 [2026-10-28 12:30:59,047] INFO : Step 2000/50000 rank1 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2138 | Classification_binary: 0.4041 | Regression_cos_phi: 0.2238 | Regression_energy: 0.2439 | Regression_eta: 0.2438 | Regression_pt: 0.2527 | Regression_sin_phi: 0.2749 | LR: 9.96e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1425s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank1 [2026-10-28 12:31:00,063] INFO : Running validation on rank1 +rank1 [2026-10-28 12:31:00,063] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 12:31:00,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 12:31:47,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:31:48,392] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 12:32:13,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:32:14,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:32:15,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:32:16,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:32:26,424] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:32:27,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 12:32:43,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:32:44,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:32:45,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:32:46,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:32:56,673] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 12:32:57,119] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:32:57,641] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 12:33:13,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:33:13,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:33:14,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:33:15,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:33:25,429] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 12:37:32,605] INFO : Step 2100/50000 rank1 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1593 | Classification_binary: 0.3295 | Regression_cos_phi: 0.1785 | Regression_energy: 0.2263 | Regression_eta: 0.1937 | Regression_pt: 0.2362 | Regression_sin_phi: 0.1968 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7723s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank1 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank1 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1808 | Classification_binary: 0.4302 | Regression_cos_phi: 0.2562 | Regression_energy: 0.2510 | Regression_eta: 0.2773 | Regression_pt: 0.2635 | Regression_sin_phi: 0.1911 | LR: 9.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7056s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank1 [2026-10-28 12:45:14,507] INFO : Step 2300/50000 rank1 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1953 | Classification_binary: 0.4288 | Regression_cos_phi: 0.2827 | Regression_energy: 0.2467 | Regression_eta: 0.2285 | Regression_pt: 0.2557 | Regression_sin_phi: 0.2441 | LR: 9.95e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.9967s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank1 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank1 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1614 | Classification_binary: 0.3864 | Regression_cos_phi: 0.1821 | Regression_energy: 0.2406 | Regression_eta: 0.1771 | Regression_pt: 0.2505 | Regression_sin_phi: 0.1964 | LR: 9.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4486s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank1 [2026-10-28 12:52:58,131] INFO : Step 2500/50000 rank1 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2219 | Classification_binary: 0.4559 | Regression_cos_phi: 0.2505 | Regression_energy: 0.2604 | Regression_eta: 0.1963 | Regression_pt: 0.2740 | Regression_sin_phi: 0.2230 | LR: 9.94e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4091s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank1 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank1 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2315 | Classification_binary: 0.4452 | Regression_cos_phi: 0.3009 | Regression_energy: 0.2527 | Regression_eta: 0.2005 | Regression_pt: 0.2645 | Regression_sin_phi: 0.2228 | LR: 9.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2536s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank1 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank1 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1627 | Classification_binary: 0.3449 | Regression_cos_phi: 0.1631 | Regression_energy: 0.2436 | Regression_eta: 0.2066 | Regression_pt: 0.2563 | Regression_sin_phi: 0.2250 | LR: 9.94e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0882s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank1 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank1 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1365 | Classification_binary: 0.2873 | Regression_cos_phi: 0.1685 | Regression_energy: 0.2337 | Regression_eta: 0.1536 | Regression_pt: 0.2442 | Regression_sin_phi: 0.1765 | LR: 9.93e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.1933s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank1 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank1 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.4206 | Regression_cos_phi: 0.2839 | Regression_energy: 0.2451 | Regression_eta: 0.2472 | Regression_pt: 0.2576 | Regression_sin_phi: 0.2611 | LR: 9.93e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.1937s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank1 [2026-10-28 13:12:09,607] INFO : Step 3000/50000 rank1 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1751 | Classification_binary: 0.3910 | Regression_cos_phi: 0.2347 | Regression_energy: 0.2575 | Regression_eta: 0.1835 | Regression_pt: 0.2675 | Regression_sin_phi: 0.2055 | LR: 9.92e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5291s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank1 [2026-10-28 13:12:10,495] INFO : Running validation on rank1 +rank1 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:13:02,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:13:03,433] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 13:13:30,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:13:31,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:13:31,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:13:32,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:13:42,925] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 13:13:43,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:13:44,105] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 13:14:00,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:14:00,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:14:01,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:14:02,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:14:12,845] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:14:14,292] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 13:14:29,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:14:30,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:14:31,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:14:31,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:14:41,707] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 13:19:02,741] INFO : Step 3100/50000 rank1 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1280 | Classification_binary: 0.2788 | Regression_cos_phi: 0.1816 | Regression_energy: 0.2398 | Regression_eta: 0.1139 | Regression_pt: 0.2558 | Regression_sin_phi: 0.1283 | LR: 9.91e-04 | DataLoad Time: 0.0230s | Model Forward Time: 5.5818s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank1 [2026-10-28 13:23:00,673] INFO : Step 3200/50000 rank1 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1913 | Classification_binary: 0.4306 | Regression_cos_phi: 0.2795 | Regression_energy: 0.2089 | Regression_eta: 0.2029 | Regression_pt: 0.2185 | Regression_sin_phi: 0.2410 | LR: 9.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8522s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank1 [2026-10-28 13:26:54,113] INFO : Step 3300/50000 rank1 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1884 | Classification_binary: 0.3913 | Regression_cos_phi: 0.1896 | Regression_energy: 0.1908 | Regression_eta: 0.1780 | Regression_pt: 0.1996 | Regression_sin_phi: 0.1853 | LR: 9.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7322s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank1 [2026-10-28 13:30:48,596] INFO : Step 3400/50000 rank1 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1692 | Classification_binary: 0.3395 | Regression_cos_phi: 0.2669 | Regression_energy: 0.2465 | Regression_eta: 0.1862 | Regression_pt: 0.2580 | Regression_sin_phi: 0.1796 | LR: 9.90e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0797s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank1 [2026-10-28 13:34:47,771] INFO : Step 3500/50000 rank1 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1767 | Classification_binary: 0.3631 | Regression_cos_phi: 0.2236 | Regression_energy: 0.2196 | Regression_eta: 0.2558 | Regression_pt: 0.2292 | Regression_sin_phi: 0.2036 | LR: 9.89e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.6682s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank1 [2026-10-28 13:38:46,123] INFO : Step 3600/50000 rank1 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2094 | Classification_binary: 0.4896 | Regression_cos_phi: 0.3195 | Regression_energy: 0.2104 | Regression_eta: 0.2357 | Regression_pt: 0.2211 | Regression_sin_phi: 0.3152 | LR: 9.89e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1035s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank1 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank1 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1579 | Classification_binary: 0.3397 | Regression_cos_phi: 0.1703 | Regression_energy: 0.2355 | Regression_eta: 0.1617 | Regression_pt: 0.2440 | Regression_sin_phi: 0.1892 | LR: 9.88e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2044s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank1 [2026-10-28 13:46:34,311] INFO : Step 3800/50000 rank1 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1742 | Classification_binary: 0.3356 | Regression_cos_phi: 0.2478 | Regression_energy: 0.1668 | Regression_eta: 0.1716 | Regression_pt: 0.1783 | Regression_sin_phi: 0.2487 | LR: 9.87e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9862s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank1 [2026-10-28 13:50:23,156] INFO : Step 3900/50000 rank1 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1832 | Classification_binary: 0.4099 | Regression_cos_phi: 0.2275 | Regression_energy: 0.2516 | Regression_eta: 0.2155 | Regression_pt: 0.2628 | Regression_sin_phi: 0.2108 | LR: 9.87e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2420s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank1 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank1 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1518 | Classification_binary: 0.3864 | Regression_cos_phi: 0.2101 | Regression_energy: 0.2046 | Regression_eta: 0.1758 | Regression_pt: 0.2142 | Regression_sin_phi: 0.2199 | LR: 9.86e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5798s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank1 [2026-10-28 13:54:12,843] INFO : Running validation on rank1 +rank1 [2026-10-28 13:54:12,843] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:54:12,843] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:55:03,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 13:55:33,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:55:34,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:55:34,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:55:35,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:55:45,916] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:55:46,361] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:55:46,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 13:56:02,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:56:03,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:56:04,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:56:05,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:56:15,758] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:56:16,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:56:16,986] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 13:56:32,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:56:32,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:56:33,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:56:34,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:56:44,210] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank1 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1639 | Classification_binary: 0.2938 | Regression_cos_phi: 0.2275 | Regression_energy: 0.2418 | Regression_eta: 0.2119 | Regression_pt: 0.2492 | Regression_sin_phi: 0.2342 | LR: 9.85e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9533s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank1 [2026-10-28 14:04:50,728] INFO : Step 4200/50000 rank1 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2122 | Classification_binary: 0.3979 | Regression_cos_phi: 0.2969 | Regression_energy: 0.2138 | Regression_eta: 0.2306 | Regression_pt: 0.2264 | Regression_sin_phi: 0.2596 | LR: 9.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4858s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank1 [2026-10-28 14:08:42,270] INFO : Step 4300/50000 rank1 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1818 | Classification_binary: 0.4704 | Regression_cos_phi: 0.2598 | Regression_energy: 0.2040 | Regression_eta: 0.2383 | Regression_pt: 0.2156 | Regression_sin_phi: 0.2987 | LR: 9.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1080s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank1 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank1 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1339 | Classification_binary: 0.2996 | Regression_cos_phi: 0.2005 | Regression_energy: 0.1777 | Regression_eta: 0.1881 | Regression_pt: 0.1814 | Regression_sin_phi: 0.2120 | LR: 9.83e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.7773s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank1 [2026-10-28 14:16:26,826] INFO : Step 4500/50000 rank1 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1324 | Classification_binary: 0.2910 | Regression_cos_phi: 0.2165 | Regression_energy: 0.2187 | Regression_eta: 0.1856 | Regression_pt: 0.2251 | Regression_sin_phi: 0.1536 | LR: 9.82e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4662s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank1 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank1 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.4588 | Regression_cos_phi: 0.2412 | Regression_energy: 0.1926 | Regression_eta: 0.1909 | Regression_pt: 0.2017 | Regression_sin_phi: 0.2082 | LR: 9.81e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1365s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank1 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank1 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1361 | Classification_binary: 0.3456 | Regression_cos_phi: 0.1647 | Regression_energy: 0.1975 | Regression_eta: 0.1581 | Regression_pt: 0.2068 | Regression_sin_phi: 0.1760 | LR: 9.81e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3027s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank1 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank1 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1363 | Classification_binary: 0.3197 | Regression_cos_phi: 0.2039 | Regression_energy: 0.2000 | Regression_eta: 0.1343 | Regression_pt: 0.2093 | Regression_sin_phi: 0.1368 | LR: 9.80e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9652s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank1 [2026-10-28 14:32:01,139] INFO : Step 4900/50000 rank1 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1868 | Classification_binary: 0.3766 | Regression_cos_phi: 0.2166 | Regression_energy: 0.2078 | Regression_eta: 0.1762 | Regression_pt: 0.2183 | Regression_sin_phi: 0.2324 | LR: 9.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8461s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank1 [2026-10-28 14:35:52,797] INFO : Step 5000/50000 rank1 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1792 | Classification_binary: 0.3402 | Regression_cos_phi: 0.3265 | Regression_energy: 0.2242 | Regression_eta: 0.1880 | Regression_pt: 0.2382 | Regression_sin_phi: 0.2479 | LR: 9.78e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8421s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank1 [2026-10-28 14:35:53,775] INFO : Running validation on rank1 +rank1 [2026-10-28 14:35:53,775] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 14:36:42,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:36:42,707] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 14:37:08,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 14:37:10,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 14:37:11,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 14:37:11,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 14:37:21,957] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:37:22,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 14:37:38,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 14:37:39,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 14:37:40,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 14:37:41,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 14:37:51,150] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 14:37:52,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:37:52,790] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 14:38:08,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 14:38:08,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 14:38:09,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 14:38:10,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 14:38:20,272] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank1 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.2829 | Regression_cos_phi: 0.1955 | Regression_energy: 0.1803 | Regression_eta: 0.1252 | Regression_pt: 0.1894 | Regression_sin_phi: 0.1566 | LR: 9.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3393s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank1 [2026-10-28 14:46:36,779] INFO : Step 5200/50000 rank1 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1812 | Classification_binary: 0.3332 | Regression_cos_phi: 0.1858 | Regression_energy: 0.1791 | Regression_eta: 0.1875 | Regression_pt: 0.1886 | Regression_sin_phi: 0.2012 | LR: 9.76e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.9465s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank1 [2026-10-28 14:50:33,436] INFO : Step 5300/50000 rank1 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1817 | Classification_binary: 0.4225 | Regression_cos_phi: 0.2742 | Regression_energy: 0.2272 | Regression_eta: 0.1526 | Regression_pt: 0.2407 | Regression_sin_phi: 0.1997 | LR: 9.75e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.1859s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank1 [2026-10-28 14:54:36,492] INFO : Step 5400/50000 rank1 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1273 | Classification_binary: 0.3372 | Regression_cos_phi: 0.2057 | Regression_energy: 0.1710 | Regression_eta: 0.1797 | Regression_pt: 0.1780 | Regression_sin_phi: 0.1884 | LR: 9.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4929s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank1 [2026-10-28 14:58:42,458] INFO : Step 5500/50000 rank1 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1642 | Classification_binary: 0.3736 | Regression_cos_phi: 0.2732 | Regression_energy: 0.2029 | Regression_eta: 0.1819 | Regression_pt: 0.2113 | Regression_sin_phi: 0.2618 | LR: 9.73e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5913s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank1 [2026-10-28 15:02:44,093] INFO : Step 5600/50000 rank1 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1854 | Classification_binary: 0.4042 | Regression_cos_phi: 0.1896 | Regression_energy: 0.1870 | Regression_eta: 0.1775 | Regression_pt: 0.1951 | Regression_sin_phi: 0.1717 | LR: 9.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6394s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank1 [2026-10-28 15:06:32,825] INFO : Step 5700/50000 rank1 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1221 | Classification_binary: 0.3170 | Regression_cos_phi: 0.1809 | Regression_energy: 0.2040 | Regression_eta: 0.1886 | Regression_pt: 0.2114 | Regression_sin_phi: 0.1688 | LR: 9.71e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1519s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank1 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank1 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.2465 | Regression_cos_phi: 0.1969 | Regression_energy: 0.1710 | Regression_eta: 0.1365 | Regression_pt: 0.1760 | Regression_sin_phi: 0.1439 | LR: 9.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5365s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank1 [2026-10-28 15:14:48,143] INFO : Step 5900/50000 rank1 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1621 | Classification_binary: 0.3337 | Regression_cos_phi: 0.2056 | Regression_energy: 0.1957 | Regression_eta: 0.1837 | Regression_pt: 0.2053 | Regression_sin_phi: 0.1963 | LR: 9.69e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.0713s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank1 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank1 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1418 | Classification_binary: 0.3554 | Regression_cos_phi: 0.1927 | Regression_energy: 0.1850 | Regression_eta: 0.1519 | Regression_pt: 0.1912 | Regression_sin_phi: 0.1977 | LR: 9.68e-04 | DataLoad Time: 0.0146s | Model Forward Time: 6.3970s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank1 [2026-10-28 15:18:46,643] INFO : Running validation on rank1 +rank1 [2026-10-28 15:18:46,643] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 15:18:46,643] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:19:35,272] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 15:19:56,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 15:19:57,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 15:19:59,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 15:20:00,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 15:20:10,882] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:20:11,685] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 15:20:27,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 15:20:28,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 15:20:29,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 15:20:30,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 15:20:40,602] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 15:20:40,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:20:41,481] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 15:20:56,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 15:20:57,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 15:20:58,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 15:20:58,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 15:21:08,697] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 15:25:37,242] INFO : Step 6100/50000 rank1 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1232 | Classification_binary: 0.2784 | Regression_cos_phi: 0.1647 | Regression_energy: 0.1805 | Regression_eta: 0.1266 | Regression_pt: 0.1861 | Regression_sin_phi: 0.1806 | LR: 9.67e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2776s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank1 [2026-10-28 15:29:27,137] INFO : Step 6200/50000 rank1 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1409 | Classification_binary: 0.2773 | Regression_cos_phi: 0.1961 | Regression_energy: 0.2132 | Regression_eta: 0.1446 | Regression_pt: 0.2273 | Regression_sin_phi: 0.1780 | LR: 9.66e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.3966s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank1 [2026-10-28 15:33:16,837] INFO : Step 6300/50000 rank1 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1774 | Classification_binary: 0.3935 | Regression_cos_phi: 0.1877 | Regression_energy: 0.1928 | Regression_eta: 0.1702 | Regression_pt: 0.2025 | Regression_sin_phi: 0.2235 | LR: 9.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2452s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank1 [2026-10-28 15:37:09,469] INFO : Step 6400/50000 rank1 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1353 | Classification_binary: 0.3258 | Regression_cos_phi: 0.1624 | Regression_energy: 0.2140 | Regression_eta: 0.1602 | Regression_pt: 0.2214 | Regression_sin_phi: 0.1771 | LR: 9.64e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9968s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank1 [2026-10-28 15:41:11,633] INFO : Step 6500/50000 rank1 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.2545 | Regression_cos_phi: 0.1180 | Regression_energy: 0.1733 | Regression_eta: 0.1434 | Regression_pt: 0.1827 | Regression_sin_phi: 0.1630 | LR: 9.63e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6828s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank1 [2026-10-28 15:45:09,040] INFO : Step 6600/50000 rank1 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1597 | Classification_binary: 0.3506 | Regression_cos_phi: 0.1968 | Regression_energy: 0.2329 | Regression_eta: 0.1434 | Regression_pt: 0.2447 | Regression_sin_phi: 0.2278 | LR: 9.62e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2532s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank1 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank1 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1486 | Classification_binary: 0.3504 | Regression_cos_phi: 0.1574 | Regression_energy: 0.1868 | Regression_eta: 0.1555 | Regression_pt: 0.1958 | Regression_sin_phi: 0.2519 | LR: 9.61e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.0306s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank1 [2026-10-28 15:52:50,800] INFO : Step 6800/50000 rank1 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 0.2934 | Regression_cos_phi: 0.1444 | Regression_energy: 0.1709 | Regression_eta: 0.1321 | Regression_pt: 0.1756 | Regression_sin_phi: 0.1522 | LR: 9.60e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8856s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank1 [2026-10-28 15:56:38,728] INFO : Step 6900/50000 rank1 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1319 | Classification_binary: 0.3186 | Regression_cos_phi: 0.1558 | Regression_energy: 0.1944 | Regression_eta: 0.1408 | Regression_pt: 0.2018 | Regression_sin_phi: 0.1662 | LR: 9.58e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.8814s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank1 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank1 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1454 | Classification_binary: 0.3362 | Regression_cos_phi: 0.1460 | Regression_energy: 0.1926 | Regression_eta: 0.1425 | Regression_pt: 0.2010 | Regression_sin_phi: 0.2216 | LR: 9.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4722s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank1 [2026-10-28 16:00:38,671] INFO : Running validation on rank1 +rank1 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:01:32,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:01:33,504] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 16:02:00,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:02:01,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:02:01,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:02:02,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:02:12,971] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:02:14,070] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 16:02:30,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:02:31,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:02:32,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:02:33,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:02:43,676] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:02:43,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:02:44,529] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 16:02:59,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:03:00,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:03:01,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:03:01,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:03:11,854] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank1 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1258 | Classification_binary: 0.2719 | Regression_cos_phi: 0.1236 | Regression_energy: 0.1796 | Regression_eta: 0.0983 | Regression_pt: 0.1844 | Regression_sin_phi: 0.1657 | LR: 9.56e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5604s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank1 [2026-10-28 16:11:41,309] INFO : Step 7200/50000 rank1 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1307 | Classification_binary: 0.3245 | Regression_cos_phi: 0.2154 | Regression_energy: 0.1577 | Regression_eta: 0.1437 | Regression_pt: 0.1612 | Regression_sin_phi: 0.1903 | LR: 9.55e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9556s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank1 [2026-10-28 16:15:45,431] INFO : Step 7300/50000 rank1 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1349 | Classification_binary: 0.3501 | Regression_cos_phi: 0.2301 | Regression_energy: 0.1848 | Regression_eta: 0.1382 | Regression_pt: 0.1911 | Regression_sin_phi: 0.1399 | LR: 9.53e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8995s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank1 [2026-10-28 16:19:47,791] INFO : Step 7400/50000 rank1 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.3209 | Regression_cos_phi: 0.2097 | Regression_energy: 0.1717 | Regression_eta: 0.1275 | Regression_pt: 0.1802 | Regression_sin_phi: 0.1835 | LR: 9.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6025s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank1 [2026-10-28 16:23:56,336] INFO : Step 7500/50000 rank1 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.2630 | Regression_cos_phi: 0.1428 | Regression_energy: 0.1565 | Regression_eta: 0.1205 | Regression_pt: 0.1644 | Regression_sin_phi: 0.1716 | LR: 9.51e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7918s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank1 [2026-10-28 16:27:58,399] INFO : Step 7600/50000 rank1 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1927 | Classification_binary: 0.3580 | Regression_cos_phi: 0.2168 | Regression_energy: 0.1715 | Regression_eta: 0.1689 | Regression_pt: 0.1761 | Regression_sin_phi: 0.2204 | LR: 9.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6995s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank1 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank1 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1146 | Classification_binary: 0.3556 | Regression_cos_phi: 0.1727 | Regression_energy: 0.1570 | Regression_eta: 0.1550 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1579 | LR: 9.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9802s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank1 [2026-10-28 16:35:55,392] INFO : Step 7800/50000 rank1 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1376 | Classification_binary: 0.2704 | Regression_cos_phi: 0.1882 | Regression_energy: 0.1836 | Regression_eta: 0.1586 | Regression_pt: 0.1918 | Regression_sin_phi: 0.1859 | LR: 9.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2110s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank1 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank1 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1514 | Classification_binary: 0.3778 | Regression_cos_phi: 0.2128 | Regression_energy: 0.1769 | Regression_eta: 0.1765 | Regression_pt: 0.1839 | Regression_sin_phi: 0.2268 | LR: 9.46e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3417s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank1 [2026-10-28 16:44:12,790] INFO : Step 8000/50000 rank1 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1394 | Classification_binary: 0.3439 | Regression_cos_phi: 0.1689 | Regression_energy: 0.1641 | Regression_eta: 0.1431 | Regression_pt: 0.1728 | Regression_sin_phi: 0.1542 | LR: 9.44e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2448s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank1 [2026-10-28 16:44:13,936] INFO : Running validation on rank1 +rank1 [2026-10-28 16:44:13,936] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:44:13,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:45:04,007] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:45:04,563] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 16:45:30,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:45:31,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:45:32,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:45:34,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:45:44,035] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 16:45:44,904] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:45:45,473] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 16:46:01,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:46:02,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:46:03,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:46:03,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:46:13,972] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:46:15,124] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 16:46:30,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:46:31,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:46:32,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:46:32,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:46:42,892] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank1 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 0.3096 | Regression_cos_phi: 0.1573 | Regression_energy: 0.1612 | Regression_eta: 0.1365 | Regression_pt: 0.1673 | Regression_sin_phi: 0.1287 | LR: 9.43e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7753s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank1 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank1 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1328 | Classification_binary: 0.2757 | Regression_cos_phi: 0.1954 | Regression_energy: 0.1573 | Regression_eta: 0.1674 | Regression_pt: 0.1640 | Regression_sin_phi: 0.2113 | LR: 9.42e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1550s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank1 [2026-10-28 16:59:31,719] INFO : Step 8300/50000 rank1 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1525 | Classification_binary: 0.3705 | Regression_cos_phi: 0.2018 | Regression_energy: 0.1605 | Regression_eta: 0.1826 | Regression_pt: 0.1679 | Regression_sin_phi: 0.2351 | LR: 9.40e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.9198s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank1 [2026-10-28 17:03:26,316] INFO : Step 8400/50000 rank1 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3650 | Regression_cos_phi: 0.2012 | Regression_energy: 0.1603 | Regression_eta: 0.1840 | Regression_pt: 0.1682 | Regression_sin_phi: 0.2152 | LR: 9.39e-04 | DataLoad Time: 0.0233s | Model Forward Time: 6.0447s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank1 [2026-10-28 17:07:18,317] INFO : Step 8500/50000 rank1 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2293 | Regression_cos_phi: 0.1155 | Regression_energy: 0.1545 | Regression_eta: 0.0839 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0904 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4980s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank1 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank1 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3704 | Regression_cos_phi: 0.1942 | Regression_energy: 0.1725 | Regression_eta: 0.1612 | Regression_pt: 0.1785 | Regression_sin_phi: 0.1784 | LR: 9.36e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.1697s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank1 [2026-10-28 17:15:44,851] INFO : Step 8700/50000 rank1 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1306 | Classification_binary: 0.3504 | Regression_cos_phi: 0.1942 | Regression_energy: 0.1473 | Regression_eta: 0.1668 | Regression_pt: 0.1524 | Regression_sin_phi: 0.1770 | LR: 9.34e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2133s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank1 [2026-10-28 17:19:40,247] INFO : Step 8800/50000 rank1 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1371 | Classification_binary: 0.2502 | Regression_cos_phi: 0.1437 | Regression_energy: 0.1333 | Regression_eta: 0.1172 | Regression_pt: 0.1374 | Regression_sin_phi: 0.1367 | LR: 9.33e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.4431s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank1 [2026-10-28 17:23:38,392] INFO : Step 8900/50000 rank1 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3140 | Regression_cos_phi: 0.2153 | Regression_energy: 0.1577 | Regression_eta: 0.1777 | Regression_pt: 0.1628 | Regression_sin_phi: 0.2437 | LR: 9.31e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6541s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank1 [2026-10-28 17:27:33,238] INFO : Step 9000/50000 rank1 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1118 | Classification_binary: 0.3320 | Regression_cos_phi: 0.1671 | Regression_energy: 0.1575 | Regression_eta: 0.1134 | Regression_pt: 0.1642 | Regression_sin_phi: 0.1596 | LR: 9.30e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8177s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank1 [2026-10-28 17:27:34,332] INFO : Running validation on rank1 +rank1 [2026-10-28 17:27:34,332] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 17:28:24,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:28:25,266] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 17:28:52,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 17:28:53,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 17:28:54,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 17:28:54,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 17:29:04,992] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:29:06,070] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 17:29:22,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 17:29:23,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 17:29:24,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 17:29:26,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 17:29:36,105] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 17:29:37,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:29:37,692] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 17:29:53,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 17:29:53,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 17:29:54,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 17:29:55,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 17:30:05,020] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 17:34:23,898] INFO : Step 9100/50000 rank1 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2760 | Regression_cos_phi: 0.1651 | Regression_energy: 0.1263 | Regression_eta: 0.1298 | Regression_pt: 0.1326 | Regression_sin_phi: 0.1364 | LR: 9.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1044s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank1 [2026-10-28 17:38:24,814] INFO : Step 9200/50000 rank1 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 0.2699 | Regression_cos_phi: 0.1606 | Regression_energy: 0.1581 | Regression_eta: 0.1198 | Regression_pt: 0.1635 | Regression_sin_phi: 0.1505 | LR: 9.27e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3261s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank1 [2026-10-28 17:42:44,641] INFO : Step 9300/50000 rank1 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1221 | Classification_binary: 0.3279 | Regression_cos_phi: 0.1812 | Regression_energy: 0.1462 | Regression_eta: 0.1466 | Regression_pt: 0.1577 | Regression_sin_phi: 0.1540 | LR: 9.25e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.8269s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank1 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank1 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2739 | Regression_cos_phi: 0.1349 | Regression_energy: 0.1665 | Regression_eta: 0.1052 | Regression_pt: 0.1759 | Regression_sin_phi: 0.1330 | LR: 9.24e-04 | DataLoad Time: 0.0100s | Model Forward Time: 4.4349s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank1 [2026-10-28 17:51:11,693] INFO : Step 9500/50000 rank1 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2826 | Regression_cos_phi: 0.1604 | Regression_energy: 0.1745 | Regression_eta: 0.1066 | Regression_pt: 0.1808 | Regression_sin_phi: 0.1273 | LR: 9.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.2531s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank1 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank1 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2558 | Regression_cos_phi: 0.1242 | Regression_energy: 0.1331 | Regression_eta: 0.1335 | Regression_pt: 0.1380 | Regression_sin_phi: 0.1109 | LR: 9.21e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8388s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank1 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank1 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.3115 | Regression_cos_phi: 0.1518 | Regression_energy: 0.1262 | Regression_eta: 0.1459 | Regression_pt: 0.1307 | Regression_sin_phi: 0.1493 | LR: 9.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9952s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank1 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank1 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.2829 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1185 | Regression_eta: 0.1294 | Regression_pt: 0.1221 | Regression_sin_phi: 0.0997 | LR: 9.17e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6870s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank1 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank1 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1132 | Classification_binary: 0.2919 | Regression_cos_phi: 0.2124 | Regression_energy: 0.1449 | Regression_eta: 0.1375 | Regression_pt: 0.1506 | Regression_sin_phi: 0.2071 | LR: 9.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1755s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank1 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank1 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.3410 | Regression_cos_phi: 0.1136 | Regression_energy: 0.1494 | Regression_eta: 0.0948 | Regression_pt: 0.1563 | Regression_sin_phi: 0.1708 | LR: 9.14e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1273s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank1 [2026-10-28 18:11:36,666] INFO : Running validation on rank1 +rank1 [2026-10-28 18:11:36,666] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:11:36,666] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:12:31,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:12:31,989] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 18:12:58,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:12:59,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:13:00,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:13:01,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:13:11,326] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:13:12,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:13:12,654] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 18:13:30,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:13:32,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:13:35,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:13:37,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:13:47,466] INFO : Time taken to make predictions on device 1 is: 0.34 min +rank1 [2026-10-28 18:13:48,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:13:49,086] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 18:14:04,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:14:05,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:14:05,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:14:06,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:14:16,678] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 18:18:32,574] INFO : Step 10100/50000 rank1 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2826 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1333 | Regression_eta: 0.0777 | Regression_pt: 0.1388 | Regression_sin_phi: 0.1134 | LR: 9.12e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2350s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank1 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank1 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.2276 | Regression_cos_phi: 0.1328 | Regression_energy: 0.1279 | Regression_eta: 0.1049 | Regression_pt: 0.1327 | Regression_sin_phi: 0.1046 | LR: 9.11e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5699s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank1 [2026-10-28 18:26:52,298] INFO : Step 10300/50000 rank1 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 0.3345 | Regression_cos_phi: 0.1589 | Regression_energy: 0.1479 | Regression_eta: 0.0858 | Regression_pt: 0.1530 | Regression_sin_phi: 0.1470 | LR: 9.09e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.3475s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank1 [2026-10-28 18:31:09,157] INFO : Step 10400/50000 rank1 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.3194 | Regression_cos_phi: 0.1398 | Regression_energy: 0.1334 | Regression_eta: 0.0905 | Regression_pt: 0.1375 | Regression_sin_phi: 0.1391 | LR: 9.07e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9069s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank1 [2026-10-28 18:35:25,945] INFO : Step 10500/50000 rank1 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2450 | Regression_cos_phi: 0.0868 | Regression_energy: 0.1192 | Regression_eta: 0.0929 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0952 | LR: 9.06e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.8287s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank1 [2026-10-28 18:39:26,893] INFO : Step 10600/50000 rank1 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2574 | Regression_cos_phi: 0.1173 | Regression_energy: 0.1182 | Regression_eta: 0.0812 | Regression_pt: 0.1266 | Regression_sin_phi: 0.1163 | LR: 9.04e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2173s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank1 [2026-10-28 18:43:27,489] INFO : Step 10700/50000 rank1 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1220 | Classification_binary: 0.3347 | Regression_cos_phi: 0.1663 | Regression_energy: 0.1340 | Regression_eta: 0.1223 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1238 | LR: 9.02e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2753s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank1 [2026-10-28 18:47:34,478] INFO : Step 10800/50000 rank1 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2384 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1145 | Regression_eta: 0.0539 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0799 | LR: 9.00e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.3727s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank1 [2026-10-28 18:51:51,745] INFO : Step 10900/50000 rank1 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.2651 | Regression_cos_phi: 0.1813 | Regression_energy: 0.1242 | Regression_eta: 0.1418 | Regression_pt: 0.1290 | Regression_sin_phi: 0.1286 | LR: 8.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6038s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank1 [2026-10-28 18:55:59,247] INFO : Step 11000/50000 rank1 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.3334 | Regression_cos_phi: 0.1381 | Regression_energy: 0.1332 | Regression_eta: 0.0975 | Regression_pt: 0.1388 | Regression_sin_phi: 0.1238 | LR: 8.97e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8665s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank1 [2026-10-28 18:56:00,216] INFO : Running validation on rank1 +rank1 [2026-10-28 18:56:00,216] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:56:00,216] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:56:54,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:56:55,375] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 18:57:21,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:57:22,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:57:23,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:57:24,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:57:34,759] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:57:35,271] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:57:35,835] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 18:57:52,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:57:53,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:57:54,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:57:55,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:58:05,326] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:58:06,769] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 18:58:22,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:58:23,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:58:23,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:58:24,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:58:34,360] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 19:03:04,100] INFO : Step 11100/50000 rank1 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2624 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1176 | Regression_eta: 0.0678 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1206 | LR: 8.95e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4761s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank1 [2026-10-28 19:07:10,330] INFO : Step 11200/50000 rank1 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2211 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1371 | Regression_eta: 0.1117 | Regression_pt: 0.1433 | Regression_sin_phi: 0.0997 | LR: 8.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8088s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank1 [2026-10-28 19:11:04,857] INFO : Step 11300/50000 rank1 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.2988 | Regression_cos_phi: 0.1459 | Regression_energy: 0.1200 | Regression_eta: 0.1013 | Regression_pt: 0.1244 | Regression_sin_phi: 0.1448 | LR: 8.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2486s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank1 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank1 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.3351 | Regression_cos_phi: 0.1174 | Regression_energy: 0.1336 | Regression_eta: 0.1117 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1200 | LR: 8.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0026s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank1 [2026-10-28 19:18:58,943] INFO : Step 11500/50000 rank1 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2490 | Regression_cos_phi: 0.0748 | Regression_energy: 0.1073 | Regression_eta: 0.0729 | Regression_pt: 0.1130 | Regression_sin_phi: 0.1080 | LR: 8.88e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.4538s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank1 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank1 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2780 | Regression_cos_phi: 0.1143 | Regression_energy: 0.1120 | Regression_eta: 0.0872 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0749 | LR: 8.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8318s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank1 [2026-10-28 19:26:57,457] INFO : Step 11700/50000 rank1 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.3508 | Regression_cos_phi: 0.1360 | Regression_energy: 0.1078 | Regression_eta: 0.0968 | Regression_pt: 0.1135 | Regression_sin_phi: 0.1270 | LR: 8.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1776s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank1 [2026-10-28 19:30:54,021] INFO : Step 11800/50000 rank1 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2406 | Regression_cos_phi: 0.1097 | Regression_energy: 0.1044 | Regression_eta: 0.0946 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0934 | LR: 8.82e-04 | DataLoad Time: 0.0258s | Model Forward Time: 6.0624s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank1 [2026-10-28 19:34:55,960] INFO : Step 11900/50000 rank1 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.2651 | Regression_cos_phi: 0.1619 | Regression_energy: 0.1118 | Regression_eta: 0.1060 | Regression_pt: 0.1192 | Regression_sin_phi: 0.1047 | LR: 8.80e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9518s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank1 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank1 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2766 | Regression_cos_phi: 0.0939 | Regression_energy: 0.1168 | Regression_eta: 0.0783 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0965 | LR: 8.78e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3557s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank1 [2026-10-28 19:39:04,975] INFO : Running validation on rank1 +rank1 [2026-10-28 19:39:04,975] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 19:39:04,975] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 19:39:55,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:39:55,582] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 19:40:21,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 19:40:22,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 19:40:23,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 19:40:24,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 19:40:34,370] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 19:40:34,825] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:40:35,407] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 19:40:51,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 19:40:52,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 19:40:53,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 19:40:54,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 19:41:04,176] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 19:41:05,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:41:05,891] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 19:41:21,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 19:41:22,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 19:41:22,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 19:41:23,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 19:41:33,447] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 19:45:57,645] INFO : Step 12100/50000 rank1 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2824 | Regression_cos_phi: 0.1103 | Regression_energy: 0.1183 | Regression_eta: 0.0757 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1018 | LR: 8.76e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8596s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank1 [2026-10-28 19:50:02,390] INFO : Step 12200/50000 rank1 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2503 | Regression_cos_phi: 0.0647 | Regression_energy: 0.1312 | Regression_eta: 0.0660 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0840 | LR: 8.74e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.9610s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank1 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank1 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0945 | Classification_binary: 0.2799 | Regression_cos_phi: 0.1228 | Regression_energy: 0.1110 | Regression_eta: 0.0960 | Regression_pt: 0.1158 | Regression_sin_phi: 0.1410 | LR: 8.72e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4594s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank1 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank1 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.3230 | Regression_cos_phi: 0.1164 | Regression_energy: 0.1047 | Regression_eta: 0.0764 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0992 | LR: 8.70e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9476s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank1 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank1 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0969 | Regression_eta: 0.0773 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0741 | LR: 8.68e-04 | DataLoad Time: 0.0248s | Model Forward Time: 4.6922s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank1 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank1 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2782 | Regression_cos_phi: 0.1083 | Regression_energy: 0.1192 | Regression_eta: 0.0786 | Regression_pt: 0.1261 | Regression_sin_phi: 0.1004 | LR: 8.66e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.9519s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank1 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank1 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2929 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0978 | Regression_eta: 0.0654 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0809 | LR: 8.64e-04 | DataLoad Time: 0.0090s | Model Forward Time: 5.2586s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank1 [2026-10-28 20:14:52,958] INFO : Step 12800/50000 rank1 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0917 | Classification_binary: 0.2909 | Regression_cos_phi: 0.1205 | Regression_energy: 0.1037 | Regression_eta: 0.0959 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0913 | LR: 8.62e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8541s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank1 [2026-10-28 20:19:01,948] INFO : Step 12900/50000 rank1 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2294 | Regression_cos_phi: 0.1295 | Regression_energy: 0.1004 | Regression_eta: 0.0719 | Regression_pt: 0.1030 | Regression_sin_phi: 0.1180 | LR: 8.60e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5183s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank1 [2026-10-28 20:23:25,500] INFO : Step 13000/50000 rank1 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2730 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0991 | Regression_eta: 0.0792 | Regression_pt: 0.1028 | Regression_sin_phi: 0.1224 | LR: 8.58e-04 | DataLoad Time: 0.0155s | Model Forward Time: 10.3336s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank1 [2026-10-28 20:23:26,427] INFO : Running validation on rank1 +rank1 [2026-10-28 20:23:26,427] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 20:23:26,427] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 20:24:18,600] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 20:24:45,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 20:24:46,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 20:24:47,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 20:24:48,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 20:24:58,334] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 20:25:04,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:25:04,756] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 20:25:20,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 20:25:21,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 20:25:22,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 20:25:23,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 20:25:33,152] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:25:34,724] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 20:25:50,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 20:25:51,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 20:25:51,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 20:25:52,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 20:26:02,474] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 20:30:28,057] INFO : Step 13100/50000 rank1 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2535 | Regression_cos_phi: 0.1145 | Regression_energy: 0.1086 | Regression_eta: 0.0733 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0581 | LR: 8.56e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0218s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank1 [2026-10-28 20:34:38,789] INFO : Step 13200/50000 rank1 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0737 | Regression_energy: 0.1093 | Regression_eta: 0.0738 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0787 | LR: 8.54e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0399s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank1 [2026-10-28 20:38:50,948] INFO : Step 13300/50000 rank1 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2561 | Regression_cos_phi: 0.0790 | Regression_energy: 0.1059 | Regression_eta: 0.0856 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0967 | LR: 8.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.4884s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank1 [2026-10-28 20:43:04,349] INFO : Step 13400/50000 rank1 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0958 | Classification_binary: 0.3103 | Regression_cos_phi: 0.1113 | Regression_energy: 0.0923 | Regression_eta: 0.1066 | Regression_pt: 0.0973 | Regression_sin_phi: 0.1083 | LR: 8.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2857s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank1 [2026-10-28 20:47:13,648] INFO : Step 13500/50000 rank1 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0971 | Regression_eta: 0.0483 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0765 | LR: 8.48e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1264s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank1 [2026-10-28 20:51:26,250] INFO : Step 13600/50000 rank1 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0965 | Regression_eta: 0.0747 | Regression_pt: 0.1023 | Regression_sin_phi: 0.1002 | LR: 8.45e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8117s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank1 [2026-10-28 20:55:41,398] INFO : Step 13700/50000 rank1 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.3179 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0986 | Regression_eta: 0.0893 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1210 | LR: 8.43e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8270s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank1 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank1 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0883 | Regression_eta: 0.0590 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0727 | LR: 8.41e-04 | DataLoad Time: 0.0227s | Model Forward Time: 6.1134s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank1 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank1 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0806 | Regression_eta: 0.0843 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0857 | LR: 8.39e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.1589s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank1 [2026-10-28 21:08:24,061] INFO : Step 14000/50000 rank1 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0960 | Classification_binary: 0.2406 | Regression_cos_phi: 0.1034 | Regression_energy: 0.1057 | Regression_eta: 0.1009 | Regression_pt: 0.1113 | Regression_sin_phi: 0.1079 | LR: 8.37e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.9193s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank1 [2026-10-28 21:08:24,997] INFO : Running validation on rank1 +rank1 [2026-10-28 21:08:24,997] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:08:24,997] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:09:16,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 21:09:43,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:09:44,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:09:44,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:09:45,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:09:55,906] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:09:56,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:09:56,997] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 21:10:13,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:10:14,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:10:15,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:10:16,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:10:26,301] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:10:26,836] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 21:10:42,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:10:43,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:10:44,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:10:45,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:10:55,834] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank1 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2693 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0934 | Regression_eta: 0.1009 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0955 | LR: 8.35e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank1 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank1 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0942 | Regression_eta: 0.0613 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0768 | LR: 8.32e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.4836s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank1 [2026-10-28 21:24:03,128] INFO : Step 14300/50000 rank1 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2276 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0802 | Regression_eta: 0.0604 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0906 | LR: 8.30e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0224s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank1 [2026-10-28 21:28:09,728] INFO : Step 14400/50000 rank1 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.3136 | Regression_cos_phi: 0.1257 | Regression_energy: 0.0934 | Regression_eta: 0.0860 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1551 | LR: 8.28e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.0272s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank1 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank1 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0934 | Regression_eta: 0.0703 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0770 | LR: 8.26e-04 | DataLoad Time: 0.0212s | Model Forward Time: 6.3471s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank1 [2026-10-28 21:36:11,051] INFO : Step 14600/50000 rank1 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0905 | Regression_eta: 0.0992 | Regression_pt: 0.0972 | Regression_sin_phi: 0.1036 | LR: 8.24e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.9764s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank1 [2026-10-28 21:40:16,608] INFO : Step 14700/50000 rank1 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.3059 | Regression_cos_phi: 0.1182 | Regression_energy: 0.0990 | Regression_eta: 0.0732 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0802 | LR: 8.21e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.1375s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank1 [2026-10-28 21:44:33,910] INFO : Step 14800/50000 rank1 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0877 | Regression_energy: 0.1065 | Regression_eta: 0.1069 | Regression_pt: 0.1157 | Regression_sin_phi: 0.1335 | LR: 8.19e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3514s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank1 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank1 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0798 | Regression_eta: 0.0771 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0767 | LR: 8.17e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.3232s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank1 [2026-10-28 21:52:53,291] INFO : Step 15000/50000 rank1 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2389 | Regression_cos_phi: 0.1224 | Regression_energy: 0.1058 | Regression_eta: 0.1046 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1211 | LR: 8.15e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7239s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank1 [2026-10-28 21:52:54,209] INFO : Running validation on rank1 +rank1 [2026-10-28 21:52:54,209] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:52:54,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:53:44,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:53:44,570] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 21:54:10,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:54:11,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:54:12,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:54:13,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:54:23,764] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:54:24,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:54:24,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 21:54:40,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:54:41,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:54:42,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:54:43,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:54:53,573] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:54:55,003] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 21:55:10,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:55:10,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:55:11,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:55:12,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:55:22,326] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 21:59:40,817] INFO : Step 15100/50000 rank1 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2363 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0852 | Regression_eta: 0.0785 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0610 | LR: 8.12e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.6916s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank1 [2026-10-28 22:03:40,551] INFO : Step 15200/50000 rank1 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0772 | Regression_eta: 0.0798 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0757 | LR: 8.10e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9694s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank1 [2026-10-28 22:07:35,582] INFO : Step 15300/50000 rank1 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0804 | Regression_eta: 0.0546 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0718 | LR: 8.08e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8189s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank1 [2026-10-28 22:11:47,342] INFO : Step 15400/50000 rank1 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2377 | Regression_cos_phi: 0.1164 | Regression_energy: 0.0824 | Regression_eta: 0.1023 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0938 | LR: 8.05e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1753s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank1 [2026-10-28 22:16:06,325] INFO : Step 15500/50000 rank1 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0777 | Regression_eta: 0.0656 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0778 | LR: 8.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9204s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank1 [2026-10-28 22:20:18,325] INFO : Step 15600/50000 rank1 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0772 | Regression_eta: 0.0714 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0893 | LR: 8.01e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.4206s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank1 [2026-10-28 22:24:30,251] INFO : Step 15700/50000 rank1 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2647 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0945 | Regression_eta: 0.0657 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0847 | LR: 7.98e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.9313s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank1 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank1 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0916 | Regression_eta: 0.0646 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0866 | LR: 7.96e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3036s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank1 [2026-10-28 22:32:57,711] INFO : Step 15900/50000 rank1 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0776 | Regression_eta: 0.0565 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0781 | LR: 7.94e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.3898s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank1 [2026-10-28 22:37:10,742] INFO : Step 16000/50000 rank1 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0884 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0924 | Regression_eta: 0.0729 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0869 | LR: 7.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0617s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank1 [2026-10-28 22:37:11,680] INFO : Running validation on rank1 +rank1 [2026-10-28 22:37:11,680] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 22:37:11,680] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 22:38:03,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:38:04,031] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 22:38:30,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 22:38:31,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 22:38:32,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 22:38:33,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 22:38:43,210] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:38:43,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:38:44,323] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 22:39:00,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 22:39:01,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 22:39:03,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 22:39:04,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 22:39:14,074] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:39:15,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:39:16,104] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 22:39:31,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 22:39:32,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 22:39:32,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 22:39:33,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 22:39:43,654] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 22:44:03,865] INFO : Step 16100/50000 rank1 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2326 | Regression_cos_phi: 0.1457 | Regression_energy: 0.0778 | Regression_eta: 0.1431 | Regression_pt: 0.0828 | Regression_sin_phi: 0.1258 | LR: 7.89e-04 | DataLoad Time: 0.0091s | Model Forward Time: 4.7982s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank1 [2026-10-28 22:48:09,285] INFO : Step 16200/50000 rank1 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2256 | Regression_cos_phi: 0.1277 | Regression_energy: 0.0870 | Regression_eta: 0.0850 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0803 | LR: 7.86e-04 | DataLoad Time: 0.0228s | Model Forward Time: 3.6635s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank1 [2026-10-28 22:52:16,253] INFO : Step 16300/50000 rank1 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2204 | Regression_cos_phi: 0.1168 | Regression_energy: 0.0934 | Regression_eta: 0.0958 | Regression_pt: 0.0979 | Regression_sin_phi: 0.1188 | LR: 7.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1374s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank1 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank1 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.2420 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0818 | Regression_eta: 0.0866 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0839 | LR: 7.81e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank1 [2026-10-28 23:00:23,724] INFO : Step 16500/50000 rank1 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.2260 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0912 | Regression_eta: 0.0990 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0965 | LR: 7.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0002s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank1 [2026-10-28 23:04:26,317] INFO : Step 16600/50000 rank1 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0861 | Regression_eta: 0.0677 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0899 | LR: 7.77e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3644s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank1 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank1 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2352 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0781 | Regression_eta: 0.0620 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0959 | LR: 7.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0578s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank1 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank1 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2612 | Regression_cos_phi: 0.1092 | Regression_energy: 0.0866 | Regression_eta: 0.1115 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0925 | LR: 7.72e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.0650s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank1 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank1 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.1863 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0723 | Regression_eta: 0.0642 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0693 | LR: 7.69e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.7648s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank1 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank1 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2248 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0829 | Regression_eta: 0.0518 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0931 | LR: 7.67e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9346s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank1 [2026-10-28 23:20:37,934] INFO : Running validation on rank1 +rank1 [2026-10-28 23:20:37,934] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:20:37,934] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 23:21:28,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:21:29,099] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 23:21:55,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 23:21:57,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 23:21:59,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 23:22:01,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 23:22:11,646] INFO : Time taken to make predictions on device 1 is: 0.31 min +rank1 [2026-10-28 23:22:12,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:22:12,959] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 23:22:28,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 23:22:29,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 23:22:30,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 23:22:31,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 23:22:41,657] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 23:22:42,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:22:42,878] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 23:22:58,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 23:22:58,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 23:22:59,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 23:23:00,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 23:23:10,433] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 23:27:38,316] INFO : Step 17100/50000 rank1 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2656 | Regression_cos_phi: 0.1111 | Regression_energy: 0.0836 | Regression_eta: 0.0875 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1038 | LR: 7.64e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.1685s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank1 [2026-10-28 23:31:38,671] INFO : Step 17200/50000 rank1 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0937 | Regression_energy: 0.0745 | Regression_eta: 0.0773 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0785 | LR: 7.62e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank1 [2026-10-28 23:35:49,549] INFO : Step 17300/50000 rank1 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0735 | Regression_eta: 0.1093 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0784 | LR: 7.59e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.5218s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank1 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank1 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2607 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0912 | Regression_eta: 0.0626 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0682 | LR: 7.57e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.3517s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank1 [2026-10-28 23:43:53,974] INFO : Step 17500/50000 rank1 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0843 | Regression_eta: 0.0838 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0897 | LR: 7.54e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9633s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank1 [2026-10-28 23:48:08,874] INFO : Step 17600/50000 rank1 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0720 | Regression_eta: 0.0597 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0546 | LR: 7.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5561s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank1 [2026-10-28 23:52:17,699] INFO : Step 17700/50000 rank1 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2277 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0900 | Regression_eta: 0.0665 | Regression_pt: 0.0917 | Regression_sin_phi: 0.1015 | LR: 7.49e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8656s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank1 [2026-10-28 23:56:40,920] INFO : Step 17800/50000 rank1 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2341 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0844 | Regression_eta: 0.1133 | Regression_pt: 0.0876 | Regression_sin_phi: 0.1143 | LR: 7.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1254s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank1 [2026-10-29 00:00:47,130] INFO : Step 17900/50000 rank1 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0813 | Regression_eta: 0.0804 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0829 | LR: 7.44e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2733s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank1 [2026-10-29 00:04:53,688] INFO : Step 18000/50000 rank1 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0723 | Regression_eta: 0.0622 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0763 | LR: 7.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 13.5875s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank1 [2026-10-29 00:04:54,626] INFO : Running validation on rank1 +rank1 [2026-10-29 00:04:54,626] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 00:04:54,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:05:43,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:05:43,820] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 00:06:10,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:06:11,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:06:12,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:06:13,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:06:23,859] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:06:24,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 00:06:41,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:06:42,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:06:43,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:06:43,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:06:53,968] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 00:07:10,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:07:11,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:07:12,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:07:12,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:07:23,004] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 00:11:38,772] INFO : Step 18100/50000 rank1 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2445 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0917 | Regression_eta: 0.0766 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0773 | LR: 7.39e-04 | DataLoad Time: 0.0245s | Model Forward Time: 4.2928s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank1 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank1 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1044 | Regression_eta: 0.0655 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0917 | LR: 7.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1153s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank1 [2026-10-29 00:19:54,086] INFO : Step 18300/50000 rank1 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0826 | Regression_eta: 0.0638 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0884 | LR: 7.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8063s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank1 [2026-10-29 00:24:07,390] INFO : Step 18400/50000 rank1 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0782 | Regression_eta: 0.0692 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0784 | LR: 7.31e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4813s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank1 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank1 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2359 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0807 | Regression_eta: 0.0920 | Regression_pt: 0.0864 | Regression_sin_phi: 0.1097 | LR: 7.29e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5479s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank1 [2026-10-29 00:31:24,472] INFO : Step 18600/50000 rank1 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0895 | Regression_eta: 0.0496 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0671 | LR: 7.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1900s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank1 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank1 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2080 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0754 | Regression_eta: 0.0725 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0794 | LR: 7.24e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.9500s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank1 [2026-10-29 00:38:49,580] INFO : Step 18800/50000 rank1 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2371 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0894 | Regression_eta: 0.1007 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0847 | LR: 7.21e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4349s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank1 [2026-10-29 00:42:28,237] INFO : Step 18900/50000 rank1 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0726 | Regression_eta: 0.0428 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0684 | LR: 7.18e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0118s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank1 [2026-10-29 00:46:10,110] INFO : Step 19000/50000 rank1 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1727 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0992 | Regression_eta: 0.0642 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0656 | LR: 7.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1521s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank1 [2026-10-29 00:46:11,053] INFO : Running validation on rank1 +rank1 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:46:58,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:46:59,172] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 00:47:25,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:47:26,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:47:27,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:47:27,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:47:38,003] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 00:47:38,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:47:38,901] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 00:47:54,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:47:55,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:47:56,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:47:57,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:48:07,502] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:48:09,173] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 00:48:24,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:48:25,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:48:26,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:48:26,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:48:36,757] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank1 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2298 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0830 | Regression_eta: 0.1044 | Regression_pt: 0.0881 | Regression_sin_phi: 0.1022 | LR: 7.13e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5899s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank1 [2026-10-29 00:56:31,489] INFO : Step 19200/50000 rank1 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0741 | Regression_eta: 0.0667 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0700 | LR: 7.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7717s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank1 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank1 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0843 | Regression_eta: 0.0628 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0664 | LR: 7.08e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6449s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank1 [2026-10-29 01:03:57,848] INFO : Step 19400/50000 rank1 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2420 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0995 | Regression_eta: 0.0867 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0726 | LR: 7.05e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank1 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank1 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0876 | Regression_eta: 0.0715 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0658 | LR: 7.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0516s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank1 [2026-10-29 01:11:25,853] INFO : Step 19600/50000 rank1 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0671 | Regression_eta: 0.0505 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0691 | LR: 7.00e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.9380s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank1 [2026-10-29 01:15:28,514] INFO : Step 19700/50000 rank1 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0695 | Regression_eta: 0.0662 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0931 | LR: 6.97e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.3659s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank1 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank1 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2395 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0894 | Regression_eta: 0.0681 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0819 | LR: 6.94e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3398s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank1 [2026-10-29 01:23:26,313] INFO : Step 19900/50000 rank1 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0779 | Regression_eta: 0.0662 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0566 | LR: 6.92e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.7467s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank1 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank1 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0742 | Regression_eta: 0.0673 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0504 | LR: 6.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8747s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank1 [2026-10-29 01:27:16,847] INFO : Running validation on rank1 +rank1 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 01:28:06,613] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:28:07,179] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 01:28:33,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 01:28:34,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 01:28:35,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 01:28:36,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 01:28:46,831] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 01:28:48,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:28:48,586] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 01:29:04,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 01:29:05,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 01:29:06,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 01:29:07,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 01:29:17,609] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 01:29:18,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:29:18,549] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 01:29:34,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 01:29:34,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 01:29:35,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 01:29:36,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 01:29:46,145] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 01:33:54,476] INFO : Step 20100/50000 rank1 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2730 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0821 | Regression_eta: 0.0703 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0815 | LR: 6.86e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.4723s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank1 [2026-10-29 01:37:52,760] INFO : Step 20200/50000 rank1 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0889 | Regression_eta: 0.0577 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0664 | LR: 6.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9232s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank1 [2026-10-29 01:41:55,631] INFO : Step 20300/50000 rank1 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0839 | Regression_eta: 0.0423 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0607 | LR: 6.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6966s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank1 [2026-10-29 01:45:46,289] INFO : Step 20400/50000 rank1 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2126 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0860 | Regression_eta: 0.0570 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0932 | LR: 6.78e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1282s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank1 [2026-10-29 01:49:32,951] INFO : Step 20500/50000 rank1 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2634 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0764 | Regression_eta: 0.0875 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0962 | LR: 6.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3416s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank1 [2026-10-29 01:53:32,128] INFO : Step 20600/50000 rank1 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2136 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0901 | Regression_eta: 0.0697 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0909 | LR: 6.73e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.0694s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank1 [2026-10-29 01:57:29,940] INFO : Step 20700/50000 rank1 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0688 | Regression_eta: 0.1092 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0760 | LR: 6.70e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.6627s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank1 [2026-10-29 02:01:35,425] INFO : Step 20800/50000 rank1 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0785 | Regression_eta: 0.0533 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0803 | LR: 6.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8012s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank1 [2026-10-29 02:05:49,594] INFO : Step 20900/50000 rank1 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2200 | Regression_cos_phi: 0.0987 | Regression_energy: 0.0901 | Regression_eta: 0.0628 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0867 | LR: 6.65e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.1317s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank1 [2026-10-29 02:09:53,643] INFO : Step 21000/50000 rank1 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0929 | Regression_eta: 0.0628 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0642 | LR: 6.62e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8413s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank1 [2026-10-29 02:09:54,610] INFO : Running validation on rank1 +rank1 [2026-10-29 02:09:54,610] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:10:45,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:10:45,899] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 02:11:12,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:11:13,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:11:14,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:11:15,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:11:25,202] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:11:26,309] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 02:11:43,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:11:44,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:11:45,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:11:47,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:11:57,072] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 02:11:57,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:11:57,965] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 02:12:13,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:12:14,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:12:14,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:12:15,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:12:25,357] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 02:16:43,089] INFO : Step 21100/50000 rank1 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2278 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0890 | Regression_eta: 0.0837 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0999 | LR: 6.59e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5466s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank1 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank1 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank1 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2184 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0977 | Regression_eta: 0.0717 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0926 | LR: 6.56e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1641s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank1 [2026-10-29 02:24:40,426] INFO : Step 21300/50000 rank1 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0760 | Regression_eta: 0.0624 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0704 | LR: 6.54e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1533s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank1 [2026-10-29 02:28:35,145] INFO : Step 21400/50000 rank1 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2457 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0716 | Regression_eta: 0.0677 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0808 | LR: 6.51e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.4144s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank1 [2026-10-29 02:32:39,448] INFO : Step 21500/50000 rank1 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0687 | Regression_eta: 0.0549 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0611 | LR: 6.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7852s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank1 [2026-10-29 02:36:35,117] INFO : Step 21600/50000 rank1 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0924 | Regression_energy: 0.1175 | Regression_eta: 0.0819 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0710 | LR: 6.45e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.8431s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank1 [2026-10-29 02:40:30,747] INFO : Step 21700/50000 rank1 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2532 | Regression_cos_phi: 0.1263 | Regression_energy: 0.1054 | Regression_eta: 0.0934 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1663 | LR: 6.43e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.5975s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank1 [2026-10-29 02:44:20,672] INFO : Step 21800/50000 rank1 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2361 | Regression_cos_phi: 0.0737 | Regression_energy: 0.1044 | Regression_eta: 0.0741 | Regression_pt: 0.1086 | Regression_sin_phi: 0.1319 | LR: 6.40e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.1253s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank1 [2026-10-29 02:48:12,873] INFO : Step 21900/50000 rank1 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0941 | Regression_eta: 0.0577 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0665 | LR: 6.37e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2334s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank1 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank1 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1011 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0939 | Regression_energy: 0.1114 | Regression_eta: 0.0658 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0838 | LR: 6.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1557s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank1 [2026-10-29 02:52:09,890] INFO : Running validation on rank1 +rank1 [2026-10-29 02:52:09,890] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:52:09,890] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:52:57,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:52:58,579] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 02:53:26,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:53:27,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:53:28,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:53:29,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:53:39,599] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 02:53:40,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 02:53:56,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:53:57,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:53:58,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:53:59,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:54:09,459] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:54:10,683] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 02:54:26,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:54:26,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:54:27,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:54:28,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:54:38,233] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank1 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2809 | Regression_cos_phi: 0.1073 | Regression_energy: 0.1030 | Regression_eta: 0.1044 | Regression_pt: 0.1080 | Regression_sin_phi: 0.1119 | LR: 6.32e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4376s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank1 [2026-10-29 03:02:36,686] INFO : Step 22200/50000 rank1 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1959 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0872 | Regression_eta: 0.0711 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0654 | LR: 6.29e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6557s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank1 [2026-10-29 03:06:28,833] INFO : Step 22300/50000 rank1 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0914 | Regression_energy: 0.1121 | Regression_eta: 0.0884 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0842 | LR: 6.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8510s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank1 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank1 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2434 | Regression_cos_phi: 0.1235 | Regression_energy: 0.1014 | Regression_eta: 0.0918 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0975 | LR: 6.23e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank1 [2026-10-29 03:14:16,069] INFO : Step 22500/50000 rank1 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0894 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1086 | Regression_energy: 0.0943 | Regression_eta: 0.0793 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0955 | LR: 6.20e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9002s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank1 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank1 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1002 | Regression_eta: 0.0921 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0742 | LR: 6.18e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5791s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank1 [2026-10-29 03:22:04,588] INFO : Step 22700/50000 rank1 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2289 | Regression_cos_phi: 0.1046 | Regression_energy: 0.0966 | Regression_eta: 0.0766 | Regression_pt: 0.1022 | Regression_sin_phi: 0.1327 | LR: 6.15e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3285s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank1 [2026-10-29 03:26:06,869] INFO : Step 22800/50000 rank1 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2526 | Regression_cos_phi: 0.0839 | Regression_energy: 0.1163 | Regression_eta: 0.0790 | Regression_pt: 0.1215 | Regression_sin_phi: 0.1076 | LR: 6.12e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.2793s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank1 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank1 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2031 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0890 | Regression_eta: 0.0639 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0601 | LR: 6.09e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.6099s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank1 [2026-10-29 03:34:03,731] INFO : Step 23000/50000 rank1 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2523 | Regression_cos_phi: 0.0907 | Regression_energy: 0.1161 | Regression_eta: 0.0972 | Regression_pt: 0.1212 | Regression_sin_phi: 0.1216 | LR: 6.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6406s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank1 [2026-10-29 03:34:04,742] INFO : Running validation on rank1 +rank1 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 03:34:52,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:34:53,013] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 03:35:18,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 03:35:19,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 03:35:20,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 03:35:21,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 03:35:31,694] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 03:35:32,426] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:35:32,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 03:35:48,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 03:35:49,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 03:35:50,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 03:35:51,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 03:36:01,379] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 03:36:02,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:36:02,911] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 03:36:18,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 03:36:19,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 03:36:20,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 03:36:20,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 03:36:30,701] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 03:40:33,162] INFO : Step 23100/50000 rank1 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2681 | Regression_cos_phi: 0.1294 | Regression_energy: 0.1170 | Regression_eta: 0.1100 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0929 | LR: 6.04e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4155s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank1 [2026-10-29 03:44:14,417] INFO : Step 23200/50000 rank1 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2115 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0813 | Regression_eta: 0.0628 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0789 | LR: 6.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4836s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank1 [2026-10-29 03:47:55,308] INFO : Step 23300/50000 rank1 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0853 | Regression_energy: 0.1085 | Regression_eta: 0.1028 | Regression_pt: 0.1130 | Regression_sin_phi: 0.1187 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9633s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank1 [2026-10-29 03:51:33,797] INFO : Step 23400/50000 rank1 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2396 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0958 | Regression_eta: 0.0670 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0945 | LR: 5.95e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1982s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank1 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank1 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2412 | Regression_cos_phi: 0.0981 | Regression_energy: 0.1124 | Regression_eta: 0.0957 | Regression_pt: 0.1159 | Regression_sin_phi: 0.1194 | LR: 5.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0137s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank1 [2026-10-29 03:59:15,166] INFO : Step 23600/50000 rank1 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1634 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0957 | Regression_eta: 0.0533 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0847 | LR: 5.90e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.9385s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank1 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank1 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2031 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0984 | Regression_eta: 0.0620 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0819 | LR: 5.87e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3851s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank1 [2026-10-29 04:06:50,323] INFO : Step 23800/50000 rank1 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2472 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1120 | Regression_eta: 0.0939 | Regression_pt: 0.1156 | Regression_sin_phi: 0.1465 | LR: 5.84e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.0063s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank1 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank1 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1928 | Regression_cos_phi: 0.0870 | Regression_energy: 0.1002 | Regression_eta: 0.0683 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0863 | LR: 5.81e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3358s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank1 [2026-10-29 04:14:57,720] INFO : Step 24000/50000 rank1 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0924 | Regression_eta: 0.0627 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0588 | LR: 5.78e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1439s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank1 [2026-10-29 04:14:58,739] INFO : Running validation on rank1 +rank1 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:15:47,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:15:48,538] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 04:16:15,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:16:16,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:16:17,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:16:18,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:16:28,435] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:16:29,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 04:16:45,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:16:45,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:16:47,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:16:47,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:16:57,993] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:16:59,582] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 04:17:15,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:17:15,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:17:16,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:17:17,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:17:27,150] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 04:21:44,038] INFO : Step 24100/50000 rank1 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2533 | Regression_cos_phi: 0.0823 | Regression_energy: 0.1070 | Regression_eta: 0.0757 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0872 | LR: 5.75e-04 | DataLoad Time: 0.0203s | Model Forward Time: 7.7523s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank1 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank1 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0775 | Regression_energy: 0.1103 | Regression_eta: 0.0674 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1074 | LR: 5.73e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank1 [2026-10-29 04:29:22,271] INFO : Step 24300/50000 rank1 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0997 | Regression_energy: 0.1027 | Regression_eta: 0.0978 | Regression_pt: 0.1069 | Regression_sin_phi: 0.1457 | LR: 5.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0422s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank1 [2026-10-29 04:33:07,495] INFO : Step 24400/50000 rank1 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1154 | Regression_energy: 0.1388 | Regression_eta: 0.1110 | Regression_pt: 0.1465 | Regression_sin_phi: 0.1506 | LR: 5.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2373s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank1 [2026-10-29 04:36:57,191] INFO : Step 24500/50000 rank1 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.3014 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1119 | Regression_eta: 0.0984 | Regression_pt: 0.1184 | Regression_sin_phi: 0.1018 | LR: 5.64e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3260s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank1 [2026-10-29 04:40:40,561] INFO : Step 24600/50000 rank1 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0826 | Regression_energy: 0.1224 | Regression_eta: 0.0758 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0723 | LR: 5.61e-04 | DataLoad Time: 0.0258s | Model Forward Time: 3.9888s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank1 [2026-10-29 04:44:29,989] INFO : Step 24700/50000 rank1 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2699 | Regression_cos_phi: 0.1032 | Regression_energy: 0.1323 | Regression_eta: 0.0951 | Regression_pt: 0.1415 | Regression_sin_phi: 0.1091 | LR: 5.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5091s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank1 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank1 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0711 | Regression_energy: 0.1230 | Regression_eta: 0.0508 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0793 | LR: 5.56e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8642s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank1 [2026-10-29 04:52:00,597] INFO : Step 24900/50000 rank1 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0700 | Regression_energy: 0.1009 | Regression_eta: 0.0714 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0841 | LR: 5.53e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9791s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank1 [2026-10-29 04:55:44,333] INFO : Step 25000/50000 rank1 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1849 | Regression_cos_phi: 0.1061 | Regression_energy: 0.1208 | Regression_eta: 0.1020 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0898 | LR: 5.50e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2476s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank1 [2026-10-29 04:55:45,304] INFO : Running validation on rank1 +rank1 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:56:33,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:56:34,566] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 04:57:00,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:57:01,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:57:02,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:57:03,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:57:13,118] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 04:57:13,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:57:13,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 04:57:29,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:57:30,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:57:31,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:57:32,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:57:42,591] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:57:44,416] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 04:57:59,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:58:00,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:58:00,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:58:01,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:58:11,551] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 05:02:15,911] INFO : Step 25100/50000 rank1 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2245 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0998 | Regression_eta: 0.0803 | Regression_pt: 0.1039 | Regression_sin_phi: 0.1151 | LR: 5.47e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7064s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank1 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank1 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2520 | Regression_cos_phi: 0.1059 | Regression_energy: 0.1125 | Regression_eta: 0.0786 | Regression_pt: 0.1162 | Regression_sin_phi: 0.1025 | LR: 5.44e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2803s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank1 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank1 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.2006 | Regression_cos_phi: 0.1174 | Regression_energy: 0.0874 | Regression_eta: 0.0725 | Regression_pt: 0.0902 | Regression_sin_phi: 0.1002 | LR: 5.42e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8558s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank1 [2026-10-29 05:13:35,667] INFO : Step 25400/50000 rank1 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.3180 | Regression_cos_phi: 0.1773 | Regression_energy: 0.1367 | Regression_eta: 0.0897 | Regression_pt: 0.1445 | Regression_sin_phi: 0.1413 | LR: 5.39e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0795s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank1 [2026-10-29 05:17:27,535] INFO : Step 25500/50000 rank1 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2576 | Regression_cos_phi: 0.0964 | Regression_energy: 0.1035 | Regression_eta: 0.0767 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0793 | LR: 5.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0768s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank1 [2026-10-29 05:21:16,672] INFO : Step 25600/50000 rank1 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2392 | Regression_cos_phi: 0.0886 | Regression_energy: 0.1060 | Regression_eta: 0.0740 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0716 | LR: 5.33e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8758s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank1 [2026-10-29 05:25:00,203] INFO : Step 25700/50000 rank1 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0729 | Regression_energy: 0.1311 | Regression_eta: 0.0584 | Regression_pt: 0.1356 | Regression_sin_phi: 0.1004 | LR: 5.30e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8871s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank1 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank1 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.2923 | Regression_cos_phi: 0.1508 | Regression_energy: 0.1126 | Regression_eta: 0.0900 | Regression_pt: 0.1188 | Regression_sin_phi: 0.1187 | LR: 5.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5177s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank1 [2026-10-29 05:32:33,908] INFO : Step 25900/50000 rank1 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2528 | Regression_cos_phi: 0.0858 | Regression_energy: 0.1006 | Regression_eta: 0.0691 | Regression_pt: 0.1050 | Regression_sin_phi: 0.1086 | LR: 5.25e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8723s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank1 [2026-10-29 05:36:28,635] INFO : Step 26000/50000 rank1 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0695 | Regression_energy: 0.1132 | Regression_eta: 0.0933 | Regression_pt: 0.1136 | Regression_sin_phi: 0.1133 | LR: 5.22e-04 | DataLoad Time: 0.0177s | Model Forward Time: 9.2326s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank1 [2026-10-29 05:36:29,552] INFO : Running validation on rank1 +rank1 [2026-10-29 05:36:29,552] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 05:36:29,552] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 05:37:19,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:37:20,020] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 05:37:45,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 05:37:46,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 05:37:46,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 05:37:47,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 05:37:57,946] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 05:37:58,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 05:38:14,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 05:38:15,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 05:38:16,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 05:38:17,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 05:38:27,465] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:38:28,891] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 05:38:44,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 05:38:45,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 05:38:45,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 05:38:46,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 05:38:56,501] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 05:43:04,441] INFO : Step 26100/50000 rank1 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.2486 | Regression_cos_phi: 0.1376 | Regression_energy: 0.0890 | Regression_eta: 0.0743 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0711 | LR: 5.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8567s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank1 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank1 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2122 | Regression_cos_phi: 0.1295 | Regression_energy: 0.1119 | Regression_eta: 0.0778 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0824 | LR: 5.16e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2717s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank1 [2026-10-29 05:50:52,485] INFO : Step 26300/50000 rank1 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2298 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1226 | Regression_eta: 0.0808 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0800 | LR: 5.13e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3839s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank1 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank1 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0789 | Classification_binary: 0.2371 | Regression_cos_phi: 0.0778 | Regression_energy: 0.1195 | Regression_eta: 0.0831 | Regression_pt: 0.1246 | Regression_sin_phi: 0.1101 | LR: 5.10e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3814s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank1 [2026-10-29 05:58:42,121] INFO : Step 26500/50000 rank1 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2529 | Regression_cos_phi: 0.0898 | Regression_energy: 0.1037 | Regression_eta: 0.0720 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0847 | LR: 5.08e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4068s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank1 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank1 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0785 | Regression_energy: 0.1284 | Regression_eta: 0.0502 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0734 | LR: 5.05e-04 | DataLoad Time: 0.0135s | Model Forward Time: 4.7817s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank1 [2026-10-29 06:06:39,601] INFO : Step 26700/50000 rank1 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2224 | Regression_cos_phi: 0.0822 | Regression_energy: 0.1025 | Regression_eta: 0.0685 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1000 | LR: 5.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.6711s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank1 [2026-10-29 06:10:41,253] INFO : Step 26800/50000 rank1 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0844 | Classification_binary: 0.3029 | Regression_cos_phi: 0.0934 | Regression_energy: 0.1156 | Regression_eta: 0.1029 | Regression_pt: 0.1204 | Regression_sin_phi: 0.1318 | LR: 4.99e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2400s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank1 [2026-10-29 06:14:37,245] INFO : Step 26900/50000 rank1 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0940 | Regression_energy: 0.1306 | Regression_eta: 0.0706 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0622 | LR: 4.96e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8907s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank1 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank1 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0902 | Classification_binary: 0.2171 | Regression_cos_phi: 0.1087 | Regression_energy: 0.1265 | Regression_eta: 0.0812 | Regression_pt: 0.1307 | Regression_sin_phi: 0.1208 | LR: 4.94e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9871s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank1 [2026-10-29 06:18:43,932] INFO : Running validation on rank1 +rank1 [2026-10-29 06:18:43,932] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 06:19:32,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:19:33,147] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 06:20:00,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 06:20:01,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 06:20:02,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 06:20:03,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 06:20:13,462] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 06:20:14,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:20:14,781] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 06:20:30,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 06:20:31,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 06:20:32,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 06:20:33,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 06:20:43,851] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:20:46,294] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 06:21:01,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 06:21:02,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 06:21:03,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 06:21:03,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 06:21:13,688] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 06:25:28,749] INFO : Step 27100/50000 rank1 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2657 | Regression_cos_phi: 0.1343 | Regression_energy: 0.1266 | Regression_eta: 0.1197 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1309 | LR: 4.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7019s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank1 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank1 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2753 | Regression_cos_phi: 0.1070 | Regression_energy: 0.1246 | Regression_eta: 0.1036 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1189 | LR: 4.88e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.5748s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank1 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank1 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0857 | Regression_eta: 0.0487 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0688 | LR: 4.85e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.9971s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank1 [2026-10-29 06:37:36,741] INFO : Step 27400/50000 rank1 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2491 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1197 | Regression_eta: 0.0762 | Regression_pt: 0.1251 | Regression_sin_phi: 0.1253 | LR: 4.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5105s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank1 [2026-10-29 06:41:28,997] INFO : Step 27500/50000 rank1 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1044 | Regression_energy: 0.1061 | Regression_eta: 0.0682 | Regression_pt: 0.1119 | Regression_sin_phi: 0.1012 | LR: 4.80e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9220s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank1 [2026-10-29 06:45:23,369] INFO : Step 27600/50000 rank1 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1130 | Regression_eta: 0.0673 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0926 | LR: 4.77e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8528s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank1 [2026-10-29 06:49:16,010] INFO : Step 27700/50000 rank1 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0847 | Regression_energy: 0.1253 | Regression_eta: 0.0592 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0765 | LR: 4.74e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0198s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank1 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank1 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2813 | Regression_cos_phi: 0.1171 | Regression_energy: 0.1040 | Regression_eta: 0.1036 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1064 | LR: 4.71e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.6676s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank1 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank1 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1202 | Regression_eta: 0.0820 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1127 | LR: 4.68e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5364s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank1 [2026-10-29 07:01:10,321] INFO : Step 28000/50000 rank1 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0866 | Regression_energy: 0.1175 | Regression_eta: 0.0785 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0946 | LR: 4.66e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.3527s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank1 [2026-10-29 07:01:11,147] INFO : Running validation on rank1 +rank1 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:02:01,773] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:02:02,348] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 07:02:30,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:02:31,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:02:32,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:02:33,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:02:43,056] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:02:44,047] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 07:03:00,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:03:00,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:03:02,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:03:03,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:03:13,644] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 07:03:13,952] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:03:14,475] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 07:03:30,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:03:30,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:03:31,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:03:32,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:03:42,050] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 07:08:04,579] INFO : Step 28100/50000 rank1 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.2319 | Regression_cos_phi: 0.1252 | Regression_energy: 0.1307 | Regression_eta: 0.1084 | Regression_pt: 0.1344 | Regression_sin_phi: 0.1252 | LR: 4.63e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1502s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank1 [2026-10-29 07:11:56,589] INFO : Step 28200/50000 rank1 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2779 | Regression_cos_phi: 0.0845 | Regression_energy: 0.1230 | Regression_eta: 0.0679 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0925 | LR: 4.60e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2152s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank1 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank1 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0972 | Regression_eta: 0.0812 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1088 | LR: 4.57e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8322s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank1 [2026-10-29 07:20:03,894] INFO : Step 28400/50000 rank1 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2452 | Regression_cos_phi: 0.0986 | Regression_energy: 0.1213 | Regression_eta: 0.0863 | Regression_pt: 0.1273 | Regression_sin_phi: 0.1027 | LR: 4.55e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0883s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank1 [2026-10-29 07:24:00,201] INFO : Step 28500/50000 rank1 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2986 | Regression_cos_phi: 0.1066 | Regression_energy: 0.1180 | Regression_eta: 0.0961 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0977 | LR: 4.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1336s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank1 [2026-10-29 07:27:52,534] INFO : Step 28600/50000 rank1 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2211 | Regression_cos_phi: 0.1071 | Regression_energy: 0.1223 | Regression_eta: 0.0716 | Regression_pt: 0.1275 | Regression_sin_phi: 0.1027 | LR: 4.49e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2888s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank1 [2026-10-29 07:31:49,172] INFO : Step 28700/50000 rank1 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0923 | Regression_energy: 0.1036 | Regression_eta: 0.0691 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1029 | LR: 4.46e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0947s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank1 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank1 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2312 | Regression_cos_phi: 0.0886 | Regression_energy: 0.1152 | Regression_eta: 0.0654 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0697 | LR: 4.44e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4530s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank1 [2026-10-29 07:39:38,719] INFO : Step 28900/50000 rank1 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2571 | Regression_cos_phi: 0.0900 | Regression_energy: 0.1016 | Regression_eta: 0.0783 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0991 | LR: 4.41e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3186s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank1 [2026-10-29 07:43:43,865] INFO : Step 29000/50000 rank1 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1863 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0902 | Regression_eta: 0.0619 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0807 | LR: 4.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9068s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank1 [2026-10-29 07:43:44,819] INFO : Running validation on rank1 +rank1 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:44:33,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:44:33,713] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 07:44:59,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:45:00,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:45:01,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:45:02,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:45:12,372] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:45:12,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:45:13,376] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 07:45:29,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:45:30,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:45:31,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:45:32,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:45:42,421] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:45:43,720] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 07:45:59,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:45:59,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:46:00,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:46:01,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:46:11,200] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 07:50:42,138] INFO : Step 29100/50000 rank1 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0807 | Regression_energy: 0.1017 | Regression_eta: 0.0647 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0959 | LR: 4.35e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.6984s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank1 [2026-10-29 07:54:30,393] INFO : Step 29200/50000 rank1 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2694 | Regression_cos_phi: 0.1460 | Regression_energy: 0.0956 | Regression_eta: 0.1081 | Regression_pt: 0.0995 | Regression_sin_phi: 0.1145 | LR: 4.33e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.2662s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank1 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank1 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0896 | Regression_energy: 0.1011 | Regression_eta: 0.0645 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0840 | LR: 4.30e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9991s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank1 [2026-10-29 08:02:11,406] INFO : Step 29400/50000 rank1 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0991 | Regression_eta: 0.0516 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0862 | LR: 4.27e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5181s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank1 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank1 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.2553 | Regression_cos_phi: 0.1431 | Regression_energy: 0.1120 | Regression_eta: 0.1168 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0925 | LR: 4.24e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4987s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank1 [2026-10-29 08:09:49,626] INFO : Step 29600/50000 rank1 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2270 | Regression_cos_phi: 0.1041 | Regression_energy: 0.0951 | Regression_eta: 0.0471 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0793 | LR: 4.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7101s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank1 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank1 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0805 | Regression_eta: 0.0518 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0435 | LR: 4.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1837s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank1 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank1 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0665 | Regression_energy: 0.1016 | Regression_eta: 0.0847 | Regression_pt: 0.1056 | Regression_sin_phi: 0.1041 | LR: 4.16e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3850s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank1 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank1 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2406 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0910 | Regression_eta: 0.0794 | Regression_pt: 0.0971 | Regression_sin_phi: 0.1006 | LR: 4.14e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9960s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank1 [2026-10-29 08:24:56,387] INFO : Step 30000/50000 rank1 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0943 | Regression_eta: 0.0639 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0634 | LR: 4.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7048s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank1 [2026-10-29 08:24:57,339] INFO : Running validation on rank1 +rank1 [2026-10-29 08:24:57,339] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 08:25:45,091] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:25:45,651] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 08:26:11,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 08:26:12,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 08:26:13,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 08:26:14,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 08:26:24,187] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 08:26:24,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:26:25,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 08:26:41,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 08:26:42,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 08:26:43,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 08:26:44,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 08:26:54,094] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:26:55,058] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 08:27:10,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 08:27:11,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 08:27:12,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 08:27:12,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 08:27:22,910] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank1 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2464 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0932 | Regression_eta: 0.0700 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0716 | LR: 4.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8520s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank1 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank1 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2393 | Regression_cos_phi: 0.0844 | Regression_energy: 0.1055 | Regression_eta: 0.0790 | Regression_pt: 0.1105 | Regression_sin_phi: 0.1007 | LR: 4.06e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5327s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank1 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank1 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0972 | Regression_eta: 0.0587 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0820 | LR: 4.03e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1810s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank1 [2026-10-29 08:43:19,167] INFO : Step 30400/50000 rank1 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1767 | Regression_cos_phi: 0.1144 | Regression_energy: 0.1029 | Regression_eta: 0.0780 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0988 | LR: 4.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5062s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank1 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank1 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0931 | Regression_eta: 0.0576 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0612 | LR: 3.98e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7628s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank1 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank1 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2089 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0765 | Regression_eta: 0.0779 | Regression_pt: 0.0815 | Regression_sin_phi: 0.1051 | LR: 3.95e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5455s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank1 [2026-10-29 08:54:35,589] INFO : Step 30700/50000 rank1 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0914 | Regression_eta: 0.0569 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0723 | LR: 3.92e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2953s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank1 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank1 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0870 | Regression_energy: 0.1057 | Regression_eta: 0.0529 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0973 | LR: 3.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2064s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank1 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank1 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2242 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0916 | Regression_eta: 0.0629 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0709 | LR: 3.87e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9272s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank1 [2026-10-29 09:05:52,365] INFO : Step 31000/50000 rank1 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0838 | Regression_eta: 0.0572 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0688 | LR: 3.84e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1881s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank1 [2026-10-29 09:05:53,351] INFO : Running validation on rank1 +rank1 [2026-10-29 09:05:53,351] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:05:53,351] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:06:41,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:06:42,226] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 09:07:08,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:07:09,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:07:10,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:07:11,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:07:21,422] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:07:22,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:07:22,582] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 09:07:38,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:07:39,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:07:40,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:07:41,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:07:51,374] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:07:52,508] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 09:08:07,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:08:08,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:08:08,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:08:09,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:08:19,650] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank1 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2607 | Regression_cos_phi: 0.1078 | Regression_energy: 0.1035 | Regression_eta: 0.1003 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0917 | LR: 3.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4607s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank1 [2026-10-29 09:16:29,206] INFO : Step 31200/50000 rank1 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2271 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0820 | Regression_eta: 0.0608 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0631 | LR: 3.79e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5349s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank1 [2026-10-29 09:20:24,829] INFO : Step 31300/50000 rank1 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0911 | Regression_eta: 0.0553 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0696 | LR: 3.76e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.4676s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank1 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank1 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0848 | Regression_eta: 0.0542 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1000 | LR: 3.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8369s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank1 [2026-10-29 09:28:17,180] INFO : Step 31500/50000 rank1 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0933 | Regression_eta: 0.0525 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0558 | LR: 3.71e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.3636s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank1 [2026-10-29 09:32:16,797] INFO : Step 31600/50000 rank1 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0713 | Regression_eta: 0.0791 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0814 | LR: 3.69e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8617s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank1 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank1 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0802 | Regression_eta: 0.0672 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0900 | LR: 3.66e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1363s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank1 [2026-10-29 09:40:22,622] INFO : Step 31800/50000 rank1 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0790 | Regression_eta: 0.0565 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0748 | LR: 3.64e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3538s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank1 [2026-10-29 09:48:15,344] INFO : Step 31900/50000 rank1 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0830 | Regression_eta: 0.0592 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0737 | LR: 3.61e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8693s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank1 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank1 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0867 | Regression_eta: 0.0632 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0694 | LR: 3.58e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.1227s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank1 [2026-10-29 09:52:26,791] INFO : Running validation on rank1 +rank1 [2026-10-29 09:52:26,791] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:52:26,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:53:17,425] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 09:53:46,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:53:47,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:53:49,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:53:50,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:54:00,293] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:54:01,008] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 09:54:16,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:54:17,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:54:18,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:54:19,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:54:29,722] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:54:30,788] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 09:54:46,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:54:47,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:54:47,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:54:48,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:54:58,460] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank1 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0802 | Regression_eta: 0.0597 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0646 | LR: 3.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9475s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank1 [2026-10-29 10:03:26,458] INFO : Step 32200/50000 rank1 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2513 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0886 | Regression_eta: 0.0599 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0582 | LR: 3.53e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5220s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank1 [2026-10-29 10:07:22,792] INFO : Step 32300/50000 rank1 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1951 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0692 | Regression_eta: 0.0615 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0671 | LR: 3.51e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5885s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank1 [2026-10-29 10:11:42,985] INFO : Step 32400/50000 rank1 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0718 | Regression_eta: 0.1083 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0863 | LR: 3.48e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.0243s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank1 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank1 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2045 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0826 | Regression_eta: 0.0614 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0937 | LR: 3.46e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5252s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank1 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank1 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2309 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0947 | Regression_eta: 0.0681 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1041 | LR: 3.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4750s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank1 [2026-10-29 10:24:11,750] INFO : Step 32700/50000 rank1 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1621 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0905 | Regression_eta: 0.0508 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0721 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0504s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank1 [2026-10-29 10:28:05,808] INFO : Step 32800/50000 rank1 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0808 | Regression_eta: 0.0755 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0772 | LR: 3.38e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5512s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank1 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank1 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2435 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0861 | Regression_eta: 0.0762 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0969 | LR: 3.36e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8085s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank1 [2026-10-29 10:36:14,933] INFO : Step 33000/50000 rank1 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0803 | Regression_eta: 0.0463 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0660 | LR: 3.33e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7649s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank1 [2026-10-29 10:36:15,961] INFO : Running validation on rank1 +rank1 [2026-10-29 10:36:15,961] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 10:36:15,961] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 10:37:03,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:37:04,371] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 10:37:32,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 10:37:34,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 10:37:35,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 10:37:37,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 10:37:47,203] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:37:48,816] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 10:38:05,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 10:38:06,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 10:38:07,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 10:38:08,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 10:38:18,174] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:38:21,704] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 10:38:37,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 10:38:38,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 10:38:38,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 10:38:39,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 10:38:49,514] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 10:43:12,058] INFO : Step 33100/50000 rank1 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0716 | Regression_eta: 0.0422 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0903 | LR: 3.31e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9476s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank1 [2026-10-29 10:47:22,844] INFO : Step 33200/50000 rank1 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2274 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0946 | Regression_eta: 0.0635 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0764 | LR: 3.28e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6981s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank1 [2026-10-29 10:51:33,084] INFO : Step 33300/50000 rank1 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0960 | Regression_eta: 0.0409 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0606 | LR: 3.26e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.4500s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank1 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank1 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0771 | Regression_eta: 0.0499 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0694 | LR: 3.23e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank1 [2026-10-29 10:59:48,622] INFO : Step 33500/50000 rank1 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2434 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0950 | Regression_eta: 0.0640 | Regression_pt: 0.0985 | Regression_sin_phi: 0.1015 | LR: 3.21e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.9162s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank1 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank1 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1998 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0818 | Regression_eta: 0.0547 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0751 | LR: 3.19e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2014s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank1 [2026-10-29 11:09:08,677] INFO : Step 33700/50000 rank1 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0803 | Regression_eta: 0.0547 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0766 | LR: 3.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3167s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank1 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank1 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0752 | Regression_eta: 0.0585 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0528 | LR: 3.14e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3669s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank1 [2026-10-29 11:16:52,541] INFO : Step 33900/50000 rank1 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.2001 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0817 | Regression_eta: 0.0609 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0818 | LR: 3.11e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8152s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank1 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank1 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0943 | Regression_energy: 0.0841 | Regression_eta: 0.0487 | Regression_pt: 0.0866 | Regression_sin_phi: 0.1134 | LR: 3.09e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.9486s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank1 [2026-10-29 11:20:48,851] INFO : Running validation on rank1 +rank1 [2026-10-29 11:20:48,851] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 11:21:42,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:21:42,910] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 11:22:09,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 11:22:10,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 11:22:10,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 11:22:11,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 11:22:21,964] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:22:22,981] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 11:22:39,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 11:22:40,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 11:22:41,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 11:22:42,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 11:22:52,438] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:22:53,478] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 11:23:09,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 11:23:09,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 11:23:10,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 11:23:11,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 11:23:21,178] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 11:27:42,516] INFO : Step 34100/50000 rank1 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2152 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0733 | Regression_eta: 0.0604 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0848 | LR: 3.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.5176s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank1 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank1 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2435 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0811 | Regression_eta: 0.0621 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0709 | LR: 3.04e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7877s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank1 [2026-10-29 11:35:56,847] INFO : Step 34300/50000 rank1 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1969 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0946 | Regression_eta: 0.0597 | Regression_pt: 0.0954 | Regression_sin_phi: 0.1075 | LR: 3.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 7.2764s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank1 [2026-10-29 11:40:08,369] INFO : Step 34400/50000 rank1 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0800 | Regression_eta: 0.0292 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0477 | LR: 2.99e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.9071s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank1 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank1 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2215 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0922 | Regression_eta: 0.0760 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0684 | LR: 2.97e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.5066s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank1 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank1 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2333 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0742 | Regression_eta: 0.0698 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0890 | LR: 2.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6054s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank1 [2026-10-29 11:52:00,097] INFO : Step 34700/50000 rank1 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0681 | Regression_eta: 0.0484 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0853 | LR: 2.92e-04 | DataLoad Time: 0.0231s | Model Forward Time: 5.5782s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank1 [2026-10-29 11:55:53,803] INFO : Step 34800/50000 rank1 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0806 | Regression_eta: 0.0550 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0656 | LR: 2.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8587s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank1 [2026-10-29 11:59:58,708] INFO : Step 34900/50000 rank1 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2298 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0874 | Regression_eta: 0.0594 | Regression_pt: 0.0908 | Regression_sin_phi: 0.1019 | LR: 2.88e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1687s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank1 [2026-10-29 12:04:06,316] INFO : Step 35000/50000 rank1 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0883 | Regression_eta: 0.0539 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0600 | LR: 2.85e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4574s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank1 [2026-10-29 12:04:07,347] INFO : Running validation on rank1 +rank1 [2026-10-29 12:04:07,347] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 12:04:07,347] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 12:05:04,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:05:05,392] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 12:05:31,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 12:05:32,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 12:05:33,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 12:05:34,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 12:05:44,775] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 12:06:01,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 12:06:02,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 12:06:03,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 12:06:04,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 12:06:14,442] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 12:06:15,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:06:16,077] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 12:06:31,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 12:06:32,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 12:06:32,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 12:06:33,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 12:06:43,480] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank1 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0827 | Regression_eta: 0.0797 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0867 | LR: 2.83e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.6537s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank1 [2026-10-29 12:15:06,103] INFO : Step 35200/50000 rank1 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0715 | Regression_eta: 0.0596 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0669 | LR: 2.81e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.0433s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank1 [2026-10-29 12:18:57,446] INFO : Step 35300/50000 rank1 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0843 | Regression_eta: 0.0394 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0531 | LR: 2.79e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8175s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank1 [2026-10-29 12:23:00,202] INFO : Step 35400/50000 rank1 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0886 | Regression_eta: 0.0656 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0579 | LR: 2.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3954s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank1 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank1 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0749 | Regression_eta: 0.0565 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0570 | LR: 2.74e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2676s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank1 [2026-10-29 12:31:17,344] INFO : Step 35600/50000 rank1 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2345 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0747 | Regression_eta: 0.0582 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0904 | LR: 2.72e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1667s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank1 [2026-10-29 12:35:31,256] INFO : Step 35700/50000 rank1 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0724 | Regression_eta: 0.0551 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0629 | LR: 2.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2777s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank1 [2026-10-29 14:49:26,307] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-29 14:49:26,320] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank1 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-29 14:49:53,253] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank1 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank1 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank1 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank1 [2026-10-29 14:49:53,279] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,282] INFO : Initializing backbone layer 3 +rank1 [2026-10-29 14:49:53,282] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,285] INFO : Initializing backbone layer 4 +rank1 [2026-10-29 14:49:53,285] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,288] INFO : Initializing backbone layer 5 +rank1 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank1 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank1 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank1 [2026-10-29 14:49:53,341] INFO : Output DNNs initialization took 0.05s +rank1 [2026-10-29 14:49:53,341] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-29 14:49:53,341] INFO : jagged_attention enabled=True +rank1 [2026-10-29 14:49:53,341] INFO : flash_attn_varlen enabled=True +rank1 [2026-10-29 14:49:53,341] INFO : task_queries enabled=False +rank1 [2026-10-29 14:49:53,341] INFO : classification_norm parameters: 0 +rank1 [2026-10-29 14:49:53,341] INFO : regression_norm parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : classification_query parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank1 [2026-10-29 14:49:53,342] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-29 14:49:53,342] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_pid parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_pt parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_eta parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-29 14:49:53,342] INFO : nn_energy parameters: 0 +rank1 [2026-10-29 14:49:53,343] INFO : Total MLPF parameters: 7260244 +rank1 [2026-10-29 14:49:53,343] INFO : MLPF __init__ done +rank1 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank1 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=1 +rank1 [2026-10-29 14:49:53,361] INFO : Moved model to device rank=1 +rank1 [2026-10-29 14:49:53,362] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-29 14:49:53,684] INFO : Restoring training from step 35001 +rank1 [2026-10-29 14:49:53,690] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank1 [2026-10-29 14:49:53,690] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-29 14:49:53,909] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-29 14:49:54,026] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank1 [2026-10-29 14:49:54,026] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-29 14:49:54,027] INFO : Trainable parameters: 7260244 +rank1 [2026-10-29 14:49:54,027] INFO : Non-trainable parameters: 0 +rank1 [2026-10-29 14:49:54,027] INFO : Total parameters: 7260244 +rank1 [2026-10-29 14:49:54,030] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-29 14:49:54,030] INFO : Getting dataloaders +rank1 [2026-10-29 14:49:54,030] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:01,434] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:02,343] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:03,545] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-29 14:50:03,545] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-29 14:50:03,545] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:03,816] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:04,072] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:04,390] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-29 14:50:04,391] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-29 14:50:18,130] INFO : Got dataloaders +rank1 [2026-10-29 14:50:18,130] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank1 [2026-10-29 14:50:18,130] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank1 [2026-10-29 14:50:18,130] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank1 [2026-10-29 14:50:18,131] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank1 [2026-10-29 14:50:19,877] INFO : loader split=train rank=1 len=21108 +rank1 [2026-10-29 14:50:19,877] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-29 14:50:19,877] INFO : Starting training from step 35001 to 50000 on rank 1 of 8 +rank1 [2026-10-29 14:50:19,877] INFO : Creating train_iterator +rank1 [2026-10-29 14:50:19,877] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank1 [2026-10-29 14:50:33,980] INFO : Created train_iterator +rank1 [2026-10-29 14:54:16,363] INFO : Step 35100/50000 rank1 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0802 | Regression_eta: 0.1099 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0754 | LR: 2.83e-04 | DataLoad Time: 0.0167s | Model Forward Time: 2.8679s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank1 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank1 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0703 | Regression_eta: 0.0550 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0708 | LR: 2.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5081s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank1 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank1 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0718 | Regression_eta: 0.0442 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0501 | LR: 2.79e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5918s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank1 [2026-10-29 15:05:10,670] INFO : Step 35400/50000 rank1 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1616 | Regression_cos_phi: 0.1119 | Regression_energy: 0.0943 | Regression_eta: 0.0673 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0827 | LR: 2.76e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.4273s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank1 [2026-10-29 15:08:49,316] INFO : Step 35500/50000 rank1 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0672 | Regression_eta: 0.0426 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0602 | LR: 2.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4542s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank1 [2026-10-29 15:12:27,419] INFO : Step 35600/50000 rank1 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0792 | Regression_eta: 0.0575 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0721 | LR: 2.72e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6884s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank1 [2026-10-29 15:16:07,387] INFO : Step 35700/50000 rank1 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0751 | Regression_eta: 0.0646 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0823 | LR: 2.70e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.3525s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank1 [2026-10-29 15:19:47,616] INFO : Step 35800/50000 rank1 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1889 | Regression_cos_phi: 0.1141 | Regression_energy: 0.0695 | Regression_eta: 0.0698 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0700 | LR: 2.68e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.4895s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank1 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank1 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0857 | Regression_eta: 0.0812 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0700 | LR: 2.65e-04 | DataLoad Time: 0.0146s | Model Forward Time: 3.8634s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank1 [2026-10-29 15:27:19,182] INFO : Step 36000/50000 rank1 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0779 | Regression_eta: 0.0637 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0689 | LR: 2.63e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8297s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank1 [2026-10-29 15:27:20,113] INFO : Running validation on rank1 +rank1 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 15:28:08,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:28:08,877] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 15:28:36,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 15:28:37,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 15:28:38,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 15:28:39,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 15:28:49,262] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 15:28:49,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:28:50,427] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 15:29:06,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 15:29:06,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 15:29:07,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 15:29:08,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 15:29:18,867] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:29:20,197] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 15:29:36,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 15:29:36,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 15:29:37,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 15:29:38,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 15:29:48,168] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank1 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0870 | Regression_eta: 0.0479 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0712 | LR: 2.61e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.9077s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank1 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank1 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2325 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0938 | Regression_eta: 0.0756 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0694 | LR: 2.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank1 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank1 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.2293 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0773 | Regression_eta: 0.0571 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0626 | LR: 2.57e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5112s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank1 [2026-10-29 15:45:12,808] INFO : Step 36400/50000 rank1 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0728 | Regression_eta: 0.0614 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0660 | LR: 2.55e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.2454s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank1 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank1 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0919 | Regression_eta: 0.0467 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0720 | LR: 2.52e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0152s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank1 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank1 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0784 | Regression_eta: 0.0563 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0610 | LR: 2.50e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7032s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank1 [2026-10-29 15:56:51,301] INFO : Step 36700/50000 rank1 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0853 | Regression_eta: 0.0616 | Regression_pt: 0.0879 | Regression_sin_phi: 0.1007 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7860s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank1 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank1 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0924 | Regression_energy: 0.1077 | Regression_eta: 0.0759 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0850 | LR: 2.46e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4177s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank1 [2026-10-29 16:04:39,810] INFO : Step 36900/50000 rank1 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2206 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0957 | Regression_eta: 0.0556 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0659 | LR: 2.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7933s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank1 [2026-10-29 16:08:30,146] INFO : Step 37000/50000 rank1 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0704 | Regression_eta: 0.0527 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0789 | LR: 2.42e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9079s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank1 [2026-10-29 16:08:31,021] INFO : Running validation on rank1 +rank1 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:09:20,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:09:20,759] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 16:09:46,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:09:47,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:09:48,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:09:49,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:09:59,160] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:10:00,499] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 16:10:16,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:10:16,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:10:17,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:10:18,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:10:28,864] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:10:30,424] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 16:10:45,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:10:46,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:10:47,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:10:47,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:10:57,792] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 16:15:06,403] INFO : Step 37100/50000 rank1 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0888 | Regression_eta: 0.0486 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1129 | LR: 2.40e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8496s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank1 [2026-10-29 16:18:46,282] INFO : Step 37200/50000 rank1 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0806 | Regression_eta: 0.0485 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0606 | LR: 2.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2289s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank1 [2026-10-29 16:22:16,730] INFO : Step 37300/50000 rank1 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0873 | Regression_eta: 0.0740 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0651 | LR: 2.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5824s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank1 [2026-10-29 16:26:01,823] INFO : Step 37400/50000 rank1 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0767 | Regression_eta: 0.0552 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0666 | LR: 2.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9409s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank1 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank1 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0860 | Regression_eta: 0.0676 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0767 | LR: 2.32e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3475s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank1 [2026-10-29 16:34:14,941] INFO : Step 37600/50000 rank1 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2336 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0884 | Regression_eta: 0.0726 | Regression_pt: 0.0921 | Regression_sin_phi: 0.1063 | LR: 2.30e-04 | DataLoad Time: 0.0134s | Model Forward Time: 4.1146s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank1 [2026-10-29 16:38:00,808] INFO : Step 37700/50000 rank1 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0837 | Regression_eta: 0.1692 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0643 | LR: 2.28e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1675s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank1 [2026-10-29 16:41:51,410] INFO : Step 37800/50000 rank1 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0850 | Regression_eta: 0.0515 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0940 | LR: 2.26e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.2921s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank1 [2026-10-29 16:45:39,811] INFO : Step 37900/50000 rank1 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0784 | Regression_eta: 0.0539 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0807 | LR: 2.24e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8080s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank1 [2026-10-29 16:49:32,192] INFO : Step 38000/50000 rank1 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0797 | Regression_eta: 0.0720 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0701 | LR: 2.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8895s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank1 [2026-10-29 16:49:33,135] INFO : Running validation on rank1 +rank1 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:50:21,275] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 16:50:47,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:50:49,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:50:51,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:50:53,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:51:03,107] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 16:51:06,253] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:51:06,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 16:51:22,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:51:23,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:51:24,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:51:25,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:51:35,548] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:51:37,186] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 16:51:52,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:51:53,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:51:53,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:51:54,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:52:04,566] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 16:56:13,529] INFO : Step 38100/50000 rank1 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0862 | Regression_eta: 0.0438 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0630 | LR: 2.20e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3706s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank1 [2026-10-29 17:00:01,012] INFO : Step 38200/50000 rank1 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0838 | Regression_eta: 0.0956 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0998 | LR: 2.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8293s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank1 [2026-10-29 17:03:44,995] INFO : Step 38300/50000 rank1 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0883 | Regression_eta: 0.0584 | Regression_pt: 0.0942 | Regression_sin_phi: 0.1127 | LR: 2.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2338s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank1 [2026-10-29 17:07:28,967] INFO : Step 38400/50000 rank1 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0768 | Regression_eta: 0.0509 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0813 | LR: 2.14e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5970s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank1 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank1 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0756 | Regression_eta: 0.0457 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0647 | LR: 2.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.5381s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank1 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank1 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2143 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0728 | Regression_eta: 0.0523 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0696 | LR: 2.11e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4113s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank1 [2026-10-29 17:18:55,036] INFO : Step 38700/50000 rank1 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2045 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0841 | Regression_eta: 0.0641 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0810 | LR: 2.09e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.8742s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank1 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank1 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0711 | Regression_eta: 0.0604 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0639 | LR: 2.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5792s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank1 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank1 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0721 | Regression_eta: 0.0582 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0733 | LR: 2.05e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6656s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank1 [2026-10-29 17:30:46,441] INFO : Step 39000/50000 rank1 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0945 | Regression_eta: 0.0452 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0841 | LR: 2.03e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6471s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank1 [2026-10-29 17:30:47,443] INFO : Running validation on rank1 +rank1 [2026-10-29 17:30:47,443] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 17:30:47,443] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 17:31:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:31:36,203] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 17:32:01,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 17:32:02,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 17:32:03,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 17:32:04,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 17:32:14,791] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:32:15,856] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 17:32:32,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 17:32:32,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 17:32:33,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 17:32:34,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 17:32:44,784] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:32:45,863] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 17:33:01,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 17:33:02,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 17:33:02,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 17:33:03,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 17:33:13,508] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank1 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0677 | Regression_eta: 0.0410 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0541 | LR: 2.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6373s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank1 [2026-10-29 17:41:18,055] INFO : Step 39200/50000 rank1 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0681 | Regression_eta: 0.0546 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0619 | LR: 2.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7439s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank1 [2026-10-29 17:45:05,746] INFO : Step 39300/50000 rank1 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0911 | Regression_eta: 0.0567 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0550 | LR: 1.98e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.9367s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank1 [2026-10-29 17:49:01,211] INFO : Step 39400/50000 rank1 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0748 | Regression_eta: 0.0494 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0704 | LR: 1.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0455s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank1 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank1 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0932 | Regression_eta: 0.0550 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0476 | LR: 1.94e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank1 [2026-10-29 17:56:59,044] INFO : Step 39600/50000 rank1 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0932 | Regression_eta: 0.0578 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0659 | LR: 1.93e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank1 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank1 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0767 | Regression_eta: 0.0718 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0668 | LR: 1.91e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2339s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank1 [2026-10-29 18:04:32,041] INFO : Step 39800/50000 rank1 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0747 | Regression_eta: 0.0351 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0548 | LR: 1.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9000s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank1 [2026-10-29 18:08:19,425] INFO : Step 39900/50000 rank1 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0784 | Regression_eta: 0.0429 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0703 | LR: 1.88e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1818s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank1 [2026-10-29 18:12:05,434] INFO : Step 40000/50000 rank1 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0730 | Regression_eta: 0.0802 | Regression_pt: 0.0769 | Regression_sin_phi: 0.1058 | LR: 1.86e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9407s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank1 [2026-10-29 18:12:06,326] INFO : Running validation on rank1 +rank1 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:12:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:12:59,647] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 18:13:23,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:13:24,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:13:25,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:13:26,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:13:36,408] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:13:37,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:13:37,720] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 18:13:53,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:13:54,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:13:55,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:13:56,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:14:06,692] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:14:07,479] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 18:14:22,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:14:23,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:14:23,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:14:24,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:14:34,474] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 18:18:48,523] INFO : Step 40100/50000 rank1 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0761 | Regression_eta: 0.0526 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0684 | LR: 1.84e-04 | DataLoad Time: 0.0104s | Model Forward Time: 5.2041s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank1 [2026-10-29 18:22:51,411] INFO : Step 40200/50000 rank1 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0806 | Regression_eta: 0.0857 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0873 | LR: 1.83e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9283s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank1 [2026-10-29 18:26:53,014] INFO : Step 40300/50000 rank1 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0817 | Regression_eta: 0.0496 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0614 | LR: 1.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4856s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank1 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank1 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0713 | Regression_eta: 0.0484 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0935 | LR: 1.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0135s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank1 [2026-10-29 18:34:31,872] INFO : Step 40500/50000 rank1 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1833 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0825 | Regression_eta: 0.0523 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0809 | LR: 1.78e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.4815s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank1 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank1 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0867 | Classification_binary: 0.2246 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0775 | Regression_eta: 0.0578 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0907 | LR: 1.76e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1611s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank1 [2026-10-29 18:42:09,670] INFO : Step 40700/50000 rank1 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0899 | Regression_eta: 0.0626 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0863 | LR: 1.75e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5738s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank1 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank1 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0762 | Regression_eta: 0.0666 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0585 | LR: 1.73e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.0686s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank1 [2026-10-29 18:49:49,168] INFO : Step 40900/50000 rank1 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0738 | Regression_eta: 0.0506 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0649 | LR: 1.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6408s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank1 [2026-10-29 18:53:41,415] INFO : Step 41000/50000 rank1 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0850 | Regression_eta: 0.0472 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0568 | LR: 1.70e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4684s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank1 [2026-10-29 18:53:42,382] INFO : Running validation on rank1 +rank1 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:54:33,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:54:33,853] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 18:54:59,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:55:00,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:55:01,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:55:02,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:55:12,541] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:55:13,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:55:13,607] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 18:55:30,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:55:31,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:55:32,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:55:33,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:55:43,101] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 18:55:59,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:55:59,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:56:00,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:56:01,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:56:11,183] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 19:00:33,894] INFO : Step 41100/50000 rank1 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1927 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0773 | Regression_eta: 0.0407 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0667 | LR: 1.69e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8495s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank1 [2026-10-29 19:04:28,229] INFO : Step 41200/50000 rank1 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0754 | Regression_eta: 0.0480 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0588 | LR: 1.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9462s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank1 [2026-10-29 19:08:18,072] INFO : Step 41300/50000 rank1 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0605 | Regression_energy: 0.1163 | Regression_eta: 0.0419 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0677 | LR: 1.66e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1414s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank1 [2026-10-29 19:12:02,348] INFO : Step 41400/50000 rank1 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0728 | Regression_eta: 0.0579 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0632 | LR: 1.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9404s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank1 [2026-10-29 19:15:48,709] INFO : Step 41500/50000 rank1 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0739 | Regression_eta: 0.0561 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0675 | LR: 1.63e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.3272s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank1 [2026-10-29 19:19:48,125] INFO : Step 41600/50000 rank1 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0668 | Regression_eta: 0.0707 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0785 | LR: 1.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7652s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank1 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank1 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2169 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0866 | Regression_eta: 0.0571 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0686 | LR: 1.60e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.9475s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank1 [2026-10-29 19:27:38,848] INFO : Step 41800/50000 rank1 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0894 | Regression_eta: 0.0497 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0669 | LR: 1.58e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.3170s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank1 [2026-10-29 19:31:32,456] INFO : Step 41900/50000 rank1 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0939 | Regression_eta: 0.0475 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0766 | LR: 1.57e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.4505s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank1 [2026-10-29 19:35:32,890] INFO : Step 42000/50000 rank1 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0826 | Regression_eta: 0.0612 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0724 | LR: 1.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9401s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank1 [2026-10-29 19:35:33,870] INFO : Running validation on rank1 +rank1 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 19:36:22,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:36:23,272] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 19:36:50,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 19:36:51,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 19:36:51,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 19:36:52,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 19:37:02,946] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:37:03,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:37:04,413] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 19:37:20,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 19:37:21,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 19:37:22,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 19:37:23,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 19:37:33,014] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 19:37:33,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:37:34,030] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 19:37:49,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 19:37:50,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 19:37:50,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 19:37:51,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 19:38:01,504] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 19:42:30,041] INFO : Step 42100/50000 rank1 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0976 | Regression_eta: 0.0704 | Regression_pt: 0.1009 | Regression_sin_phi: 0.1050 | LR: 1.54e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0442s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank1 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank1 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0849 | Regression_energy: 0.1078 | Regression_eta: 0.0699 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0853 | LR: 1.53e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8374s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank1 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank1 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 19:50:51,512] INFO : Step 42300/50000 rank1 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0954 | Regression_eta: 0.0646 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0921 | LR: 1.52e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2014s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank1 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank1 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0909 | Regression_eta: 0.0434 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0480 | LR: 1.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6753s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank1 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank1 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0750 | Regression_eta: 0.0761 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0858 | LR: 1.49e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1941s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank1 [2026-10-29 20:02:59,632] INFO : Step 42600/50000 rank1 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2522 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0964 | Regression_eta: 0.0695 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0852 | LR: 1.48e-04 | DataLoad Time: 0.0149s | Model Forward Time: 8.8103s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank1 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank1 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0864 | Regression_eta: 0.0450 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0764 | LR: 1.47e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7995s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank1 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank1 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1837 | Regression_cos_phi: 0.1054 | Regression_energy: 0.0828 | Regression_eta: 0.0581 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0811 | LR: 1.45e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5857s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank1 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank1 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.2247 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0855 | Regression_eta: 0.0522 | Regression_pt: 0.0897 | Regression_sin_phi: 0.1012 | LR: 1.44e-04 | DataLoad Time: 0.0110s | Model Forward Time: 4.4351s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank1 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank1 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0850 | Regression_eta: 0.0503 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0688 | LR: 1.43e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9464s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank1 [2026-10-29 20:18:48,626] INFO : Running validation on rank1 +rank1 [2026-10-29 20:18:48,626] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 20:19:38,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:19:39,385] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 20:20:09,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 20:20:10,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 20:20:11,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 20:20:12,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 20:20:22,616] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:20:23,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:20:23,703] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 20:20:39,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 20:20:40,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 20:20:41,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 20:20:42,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 20:20:52,604] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:20:53,526] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:20:53,998] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 20:21:09,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 20:21:10,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 20:21:10,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 20:21:11,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 20:21:21,653] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank1 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1677 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1040 | Regression_eta: 0.0558 | Regression_pt: 0.1091 | Regression_sin_phi: 0.1004 | LR: 1.42e-04 | DataLoad Time: 0.0239s | Model Forward Time: 4.2850s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank1 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank1 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.1837 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0773 | Regression_eta: 0.1102 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0952 | LR: 1.40e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1806s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank1 [2026-10-29 20:33:23,251] INFO : Step 43300/50000 rank1 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0906 | Regression_eta: 0.0556 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0665 | LR: 1.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3866s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank1 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank1 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0925 | Regression_eta: 0.0485 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0676 | LR: 1.38e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5481s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank1 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank1 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0792 | Regression_eta: 0.0598 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0578 | LR: 1.37e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1838s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank1 [2026-10-29 20:44:45,321] INFO : Step 43600/50000 rank1 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2490 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0751 | Regression_eta: 0.0744 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0755 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0813s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank1 [2026-10-29 20:48:25,993] INFO : Step 43700/50000 rank1 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0676 | Regression_energy: 0.1170 | Regression_eta: 0.0491 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0865 | LR: 1.35e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.4599s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank1 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank1 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0794 | Regression_eta: 0.0572 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0643 | LR: 1.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.5350s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank1 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank1 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0894 | Regression_eta: 0.0505 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0589 | LR: 1.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3473s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank1 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank1 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2330 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0917 | Regression_eta: 0.0449 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0753 | LR: 1.32e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1961s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank1 [2026-10-29 21:00:02,712] INFO : Running validation on rank1 +rank1 [2026-10-29 21:00:02,712] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:00:57,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:00:58,023] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 21:01:24,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:01:25,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:01:26,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:01:27,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:01:37,881] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:01:38,954] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 21:01:54,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:01:55,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:01:56,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:01:57,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:02:07,532] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:02:08,957] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 21:02:24,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:02:25,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:02:25,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:02:26,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:02:36,432] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 21:06:42,682] INFO : Step 44100/50000 rank1 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0773 | Regression_eta: 0.0450 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0670 | LR: 1.31e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.1688s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank1 [2026-10-29 21:10:40,736] INFO : Step 44200/50000 rank1 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0787 | Regression_eta: 0.0570 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0615 | LR: 1.30e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9777s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank1 [2026-10-29 21:14:29,481] INFO : Step 44300/50000 rank1 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1958 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0816 | Regression_eta: 0.0509 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0642 | LR: 1.29e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1173s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank1 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank1 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0787 | Regression_eta: 0.0427 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0566 | LR: 1.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7685s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank1 [2026-10-29 21:22:16,364] INFO : Step 44500/50000 rank1 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0768 | Regression_eta: 0.0463 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0791 | LR: 1.27e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.8093s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank1 [2026-10-29 21:26:10,595] INFO : Step 44600/50000 rank1 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.2105 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0858 | Regression_eta: 0.0486 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0384 | LR: 1.26e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1489s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank1 [2026-10-29 21:30:16,075] INFO : Step 44700/50000 rank1 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0750 | Regression_eta: 0.0415 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0818 | LR: 1.25e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7000s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank1 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank1 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0849 | Regression_eta: 0.0484 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0700 | LR: 1.24e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.4083s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank1 [2026-10-29 21:38:11,005] INFO : Step 44900/50000 rank1 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0804 | Regression_eta: 0.0749 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0799 | LR: 1.23e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9541s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank1 [2026-10-29 21:42:13,644] INFO : Step 45000/50000 rank1 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2261 | Regression_cos_phi: 0.0960 | Regression_energy: 0.0980 | Regression_eta: 0.0648 | Regression_pt: 0.1030 | Regression_sin_phi: 0.1227 | LR: 1.22e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0117s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank1 [2026-10-29 21:42:14,558] INFO : Running validation on rank1 +rank1 [2026-10-29 21:42:14,558] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:42:14,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:43:06,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:43:07,218] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 21:43:32,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:43:33,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:43:34,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:43:35,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:43:45,992] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:43:47,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:43:47,829] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 21:44:03,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:44:04,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:44:05,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:44:06,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:44:16,387] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:44:17,710] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 21:44:32,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:44:33,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:44:34,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:44:34,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:44:44,972] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 21:48:55,617] INFO : Step 45100/50000 rank1 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0964 | Regression_eta: 0.0606 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0881 | LR: 1.21e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4107s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank1 [2026-10-29 21:52:45,387] INFO : Step 45200/50000 rank1 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0852 | Regression_eta: 0.0531 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0993 | LR: 1.20e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1526s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank1 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank1 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0703 | Regression_eta: 0.0540 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0797 | LR: 1.19e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0128s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank1 [2026-10-29 22:00:32,927] INFO : Step 45400/50000 rank1 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0937 | Regression_eta: 0.0598 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0405 | LR: 1.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9034s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank1 [2026-10-29 22:04:23,231] INFO : Step 45500/50000 rank1 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0828 | Regression_eta: 0.0409 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0863 | LR: 1.18e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6347s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank1 [2026-10-29 22:08:21,193] INFO : Step 45600/50000 rank1 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0897 | Regression_eta: 0.0766 | Regression_pt: 0.0951 | Regression_sin_phi: 0.1007 | LR: 1.17e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0164s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank1 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank1 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2172 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0828 | Regression_eta: 0.0772 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0789 | LR: 1.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0852s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank1 [2026-10-29 22:16:08,603] INFO : Step 45800/50000 rank1 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0867 | Regression_eta: 0.0383 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0663 | LR: 1.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6674s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank1 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank1 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0858 | Regression_eta: 0.0466 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0577 | LR: 1.15e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9602s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank1 [2026-10-29 22:24:24,449] INFO : Step 46000/50000 rank1 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.2174 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0797 | Regression_eta: 0.0574 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0521 | LR: 1.14e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.2975s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank1 [2026-10-29 22:24:25,403] INFO : Running validation on rank1 +rank1 [2026-10-29 22:24:25,403] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 22:24:25,403] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 22:25:16,575] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:25:17,231] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 22:25:46,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 22:25:47,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 22:25:48,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 22:25:49,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 22:25:59,393] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 22:25:59,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:26:00,492] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 22:26:16,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 22:26:17,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 22:26:18,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 22:26:19,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 22:26:29,861] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:26:32,187] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 22:26:47,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 22:26:48,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 22:26:49,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 22:26:49,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 22:26:59,879] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 22:31:16,603] INFO : Step 46100/50000 rank1 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0918 | Regression_eta: 0.0422 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0721 | LR: 1.13e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0428s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank1 [2026-10-29 22:35:12,809] INFO : Step 46200/50000 rank1 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0866 | Regression_eta: 0.0441 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0604 | LR: 1.13e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.9320s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank1 [2026-10-29 22:39:25,151] INFO : Step 46300/50000 rank1 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2001 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0713 | Regression_eta: 0.0556 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0749 | LR: 1.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9858s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank1 [2026-10-29 22:43:23,785] INFO : Step 46400/50000 rank1 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0834 | Regression_eta: 0.0488 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0573 | LR: 1.11e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6677s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank1 [2026-10-29 22:47:17,350] INFO : Step 46500/50000 rank1 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0621 | Regression_eta: 0.0645 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0697 | LR: 1.11e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.5775s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank1 [2026-10-29 22:51:20,583] INFO : Step 46600/50000 rank1 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.2129 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0836 | Regression_eta: 0.0475 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0817 | LR: 1.10e-04 | DataLoad Time: 0.0106s | Model Forward Time: 5.4145s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank1 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank1 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0958 | Regression_energy: 0.0980 | Regression_eta: 0.0663 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0690 | LR: 1.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0478s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank1 [2026-10-29 22:59:26,804] INFO : Step 46800/50000 rank1 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0767 | Regression_eta: 0.0399 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0492 | LR: 1.09e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.9198s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank1 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank1 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1868 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0813 | Regression_eta: 0.0639 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0734 | LR: 1.09e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0381s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank1 [2026-10-29 23:08:15,439] INFO : Step 47000/50000 rank1 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0845 | Regression_eta: 0.0639 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0770 | LR: 1.08e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2716s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank1 [2026-10-29 23:08:16,506] INFO : Running validation on rank1 +rank1 [2026-10-29 23:08:16,506] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 23:08:16,506] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 23:09:07,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:09:08,184] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 23:09:36,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 23:09:38,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 23:09:40,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 23:09:42,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 23:09:52,176] INFO : Time taken to make predictions on device 1 is: 0.31 min +rank1 [2026-10-29 23:09:55,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:09:55,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 23:10:11,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 23:10:12,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 23:10:13,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 23:10:14,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 23:10:24,296] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:10:25,936] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 23:10:40,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 23:10:41,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 23:10:42,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 23:10:42,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 23:10:52,990] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank1 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0915 | Regression_eta: 0.0673 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0991 | LR: 1.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7060s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank1 [2026-10-29 23:19:19,792] INFO : Step 47200/50000 rank1 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0673 | Regression_eta: 0.0612 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0785 | LR: 1.07e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.6922s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank1 [2026-10-29 23:23:31,492] INFO : Step 47300/50000 rank1 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0853 | Regression_eta: 0.0564 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0868 | LR: 1.06e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7150s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank1 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank1 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0870 | Regression_eta: 0.0443 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0535 | LR: 1.06e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0210s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank1 [2026-10-29 23:31:38,583] INFO : Step 47500/50000 rank1 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0830 | Regression_eta: 0.0475 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0613 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8375s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank1 [2026-10-29 23:35:34,354] INFO : Step 47600/50000 rank1 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0786 | Regression_eta: 0.0453 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0746 | LR: 1.05e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5016s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank1 [2026-10-29 23:39:49,206] INFO : Step 47700/50000 rank1 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0802 | Regression_eta: 0.0681 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0824 | LR: 1.05e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8414s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank1 [2026-10-29 23:43:53,970] INFO : Step 47800/50000 rank1 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0848 | Regression_eta: 0.0625 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0537 | LR: 1.04e-04 | DataLoad Time: 0.0121s | Model Forward Time: 4.6222s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank1 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank1 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0801 | Regression_eta: 0.0610 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0751 | LR: 1.04e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1962s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank1 [2026-10-29 23:51:38,726] INFO : Step 48000/50000 rank1 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0732 | Regression_eta: 0.0562 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0570 | LR: 1.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2963s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank1 [2026-10-29 23:51:39,727] INFO : Running validation on rank1 +rank1 [2026-10-29 23:51:39,727] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 23:51:39,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 23:52:31,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:52:31,695] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 23:52:58,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 23:52:59,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 23:53:01,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 23:53:02,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 23:53:12,344] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 23:53:13,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:53:13,953] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 23:53:29,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 23:53:30,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 23:53:31,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 23:53:32,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 23:53:42,729] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:53:43,871] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 23:53:59,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 23:53:59,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 23:54:00,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 23:54:01,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 23:54:11,144] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-30 00:00:53,531] INFO : Step 48100/50000 rank1 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0760 | Regression_eta: 0.0438 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0538 | LR: 1.03e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5294s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank1 [2026-10-30 00:04:40,583] INFO : Step 48200/50000 rank1 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0794 | Regression_eta: 0.0513 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0721 | LR: 1.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0277s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank1 [2026-10-30 00:08:31,628] INFO : Step 48300/50000 rank1 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0790 | Regression_eta: 0.0470 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0566 | LR: 1.03e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9455s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank1 [2026-10-30 00:12:19,575] INFO : Step 48400/50000 rank1 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.2037 | Regression_cos_phi: 0.1052 | Regression_energy: 0.0834 | Regression_eta: 0.0536 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0707 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3350s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank1 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank1 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0766 | Regression_eta: 0.0499 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0598 | LR: 1.02e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.5882s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank1 [2026-10-30 00:19:50,481] INFO : Step 48600/50000 rank1 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2169 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0750 | Regression_eta: 0.0648 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0818 | LR: 1.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9682s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank1 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank1 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0748 | Regression_eta: 0.0451 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0589 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0926s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank1 [2026-10-30 00:27:30,341] INFO : Step 48800/50000 rank1 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0747 | Regression_eta: 0.0481 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0580 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4239s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank1 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank1 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0736 | Regression_eta: 0.0486 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0861 | LR: 1.01e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank1 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank1 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0726 | Regression_eta: 0.0653 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0513 | LR: 1.01e-04 | DataLoad Time: 0.0095s | Model Forward Time: 5.6181s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank1 [2026-10-30 00:35:15,046] INFO : Running validation on rank1 +rank1 [2026-10-30 00:35:15,046] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:35:15,046] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:36:03,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 00:36:29,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:36:31,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:36:32,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:36:33,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:36:43,236] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:36:48,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:36:48,746] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 00:37:04,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:37:05,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:37:06,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:37:07,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:37:17,232] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:37:18,485] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 00:37:33,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:37:34,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 00:37:34,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 00:37:35,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 00:37:45,636] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-30 00:41:49,930] INFO : Step 49100/50000 rank1 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0801 | Regression_eta: 0.0297 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0447 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.6176s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank1 [2026-10-30 00:45:59,115] INFO : Step 49200/50000 rank1 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0699 | Regression_eta: 0.0489 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0457 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5917s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank1 [2026-10-30 00:49:54,420] INFO : Step 49300/50000 rank1 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0799 | Regression_eta: 0.0529 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0683 | LR: 1.00e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9862s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank1 [2026-10-30 00:53:44,253] INFO : Step 49400/50000 rank1 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0747 | Regression_eta: 0.0589 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0675 | LR: 1.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9621s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank1 [2026-10-30 00:57:39,408] INFO : Step 49500/50000 rank1 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0753 | Regression_eta: 0.0454 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0526 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0097s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank1 [2026-10-30 01:01:39,149] INFO : Step 49600/50000 rank1 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0723 | Regression_eta: 0.0507 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0789 | LR: 1.00e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.8026s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank1 [2026-10-30 01:05:30,841] INFO : Step 49700/50000 rank1 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0875 | Regression_eta: 0.0584 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0541 | LR: 1.00e-04 | DataLoad Time: 0.0104s | Model Forward Time: 4.4672s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank1 [2026-10-30 01:09:21,932] INFO : Step 49800/50000 rank1 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0763 | Regression_eta: 0.0532 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0609 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3594s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank1 [2026-10-30 01:13:13,493] INFO : Step 49900/50000 rank1 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0870 | Regression_eta: 0.0414 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0519 | LR: 1.00e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0827s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank1 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank1 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0783 | Regression_eta: 0.0639 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0835 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8270s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank1 [2026-10-30 01:17:09,618] INFO : Running validation on rank1 +rank1 [2026-10-30 01:17:09,618] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 01:17:09,618] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 01:18:02,737] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:18:03,253] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 01:18:29,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 01:18:30,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 01:18:31,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 01:18:32,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 01:18:42,013] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:18:42,590] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:18:43,092] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 01:18:59,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 01:18:59,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 01:19:00,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 01:19:01,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 01:19:11,781] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 01:19:28,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 01:19:29,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:19:29,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:19:30,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:19:40,367] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-30 01:19:41,571] INFO : Training completed. Total time on device 1: 629.362min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.2 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..699a23df72727615c16e43085d73c47f92a113d1 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.2 @@ -0,0 +1,2483 @@ +rank2 [2026-10-28 11:12:32,935] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-28 11:12:32,936] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank2 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-28 11:12:57,452] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-28 11:12:57,452] INFO : nn0 parameters: 171776 +rank2 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank2 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,456] INFO : Initializing backbone layer 1 +rank2 [2026-10-28 11:12:57,456] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 2 +rank2 [2026-10-28 11:12:57,460] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,463] INFO : Initializing backbone layer 3 +rank2 [2026-10-28 11:12:57,463] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,466] INFO : Initializing backbone layer 4 +rank2 [2026-10-28 11:12:57,466] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,469] INFO : Initializing backbone layer 5 +rank2 [2026-10-28 11:12:57,469] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:57,472] INFO : Backbone initialization took 0.02s +rank2 [2026-10-28 11:12:57,472] INFO : backbone parameters: 2374656 +rank2 [2026-10-28 11:12:57,473] INFO : Initializing output DNNs +rank2 [2026-10-28 11:12:57,522] INFO : Output DNNs initialization took 0.05s +rank2 [2026-10-28 11:12:57,522] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-28 11:12:57,523] INFO : jagged_attention enabled=True +rank2 [2026-10-28 11:12:57,523] INFO : flash_attn_varlen enabled=True +rank2 [2026-10-28 11:12:57,523] INFO : task_queries enabled=False +rank2 [2026-10-28 11:12:57,523] INFO : classification_norm parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : regression_norm parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : classification_query parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank2 [2026-10-28 11:12:57,523] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-28 11:12:57,523] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-28 11:12:57,523] INFO : nn_pid parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : nn_pt parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : nn_eta parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : nn_energy parameters: 0 +rank2 [2026-10-28 11:12:57,524] INFO : Total MLPF parameters: 7260244 +rank2 [2026-10-28 11:12:57,524] INFO : MLPF __init__ done +rank2 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank2 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=2 +rank2 [2026-10-28 11:12:57,558] INFO : Moved model to device rank=2 +rank2 [2026-10-28 11:12:57,559] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-28 11:12:57,560] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-28 11:12:57,812] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank2 [2026-10-28 11:12:57,812] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-28 11:12:57,813] INFO : Trainable parameters: 7260244 +rank2 [2026-10-28 11:12:57,813] INFO : Non-trainable parameters: 0 +rank2 [2026-10-28 11:12:57,813] INFO : Total parameters: 7260244 +rank2 [2026-10-28 11:12:57,816] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-28 11:12:57,816] INFO : Getting dataloaders +rank2 [2026-10-28 11:12:57,816] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:03,708] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:04,767] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,271] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,538] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,799] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-28 11:13:06,799] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-28 11:13:20,951] INFO : Got dataloaders +rank2 [2026-10-28 11:13:20,952] INFO : loader split=train rank=2 len=21108 +rank2 [2026-10-28 11:13:20,952] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-28 11:13:20,952] INFO : Starting training from step 1 to 50000 on rank 2 of 8 +rank2 [2026-10-28 11:13:20,952] INFO : Creating train_iterator +rank2 [2026-10-28 11:13:20,952] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-28 11:13:35,216] INFO : Created train_iterator +rank2 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank2 [2026-10-28 11:17:20,490] INFO : Step 100/50000 rank2 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8349 | Classification_binary: 0.5190 | Regression_cos_phi: 0.0071 | Regression_energy: 1.3921 | Regression_eta: 0.0168 | Regression_pt: 1.3242 | Regression_sin_phi: 0.0070 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.1321s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank2 [2026-10-28 11:20:57,690] INFO : Step 200/50000 rank2 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4402 | Classification_binary: 0.5191 | Regression_cos_phi: 0.3480 | Regression_energy: 0.3167 | Regression_eta: 0.4287 | Regression_pt: 0.3449 | Regression_sin_phi: 0.4780 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.5600s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank2 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank2 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3492 | Classification_binary: 0.4554 | Regression_cos_phi: 0.3633 | Regression_energy: 0.3027 | Regression_eta: 0.4054 | Regression_pt: 0.3054 | Regression_sin_phi: 0.4599 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 3.7584s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank2 [2026-10-28 11:28:09,551] INFO : Step 400/50000 rank2 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2864 | Classification_binary: 0.4607 | Regression_cos_phi: 0.3871 | Regression_energy: 0.3322 | Regression_eta: 0.3792 | Regression_pt: 0.3399 | Regression_sin_phi: 0.3341 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 3.5799s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank2 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank2 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3247 | Classification_binary: 0.4909 | Regression_cos_phi: 0.3230 | Regression_energy: 0.2703 | Regression_eta: 0.3377 | Regression_pt: 0.2795 | Regression_sin_phi: 0.3565 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 4.0260s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank2 [2026-10-28 11:35:32,722] INFO : Step 600/50000 rank2 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2550 | Classification_binary: 0.5191 | Regression_cos_phi: 0.3005 | Regression_energy: 0.3180 | Regression_eta: 0.2521 | Regression_pt: 0.3370 | Regression_sin_phi: 0.3031 | LR: 1.00e-03 | DataLoad Time: 0.0150s | Model Forward Time: 4.1155s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank2 [2026-10-28 11:39:09,491] INFO : Step 700/50000 rank2 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2289 | Classification_binary: 0.4198 | Regression_cos_phi: 0.3059 | Regression_energy: 0.2847 | Regression_eta: 0.3248 | Regression_pt: 0.2935 | Regression_sin_phi: 0.2387 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 3.3991s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank2 [2026-10-28 11:42:49,057] INFO : Step 800/50000 rank2 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2664 | Classification_binary: 0.4443 | Regression_cos_phi: 0.2940 | Regression_energy: 0.3278 | Regression_eta: 0.3636 | Regression_pt: 0.3518 | Regression_sin_phi: 0.2882 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.6492s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank2 [2026-10-28 11:46:31,920] INFO : Step 900/50000 rank2 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2086 | Classification_binary: 0.4908 | Regression_cos_phi: 0.2677 | Regression_energy: 0.2762 | Regression_eta: 0.2426 | Regression_pt: 0.2879 | Regression_sin_phi: 0.2635 | LR: 9.99e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.5695s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank2 [2026-10-28 11:50:09,949] INFO : Step 1000/50000 rank2 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2019 | Classification_binary: 0.4004 | Regression_cos_phi: 0.1955 | Regression_energy: 0.3255 | Regression_eta: 0.1997 | Regression_pt: 0.3358 | Regression_sin_phi: 0.1997 | LR: 9.99e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4173s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank2 [2026-10-28 11:50:10,910] INFO : Running validation on rank2 +rank2 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 11:50:58,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:50:59,086] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 11:51:19,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 11:51:20,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 11:51:21,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 11:51:22,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 11:51:32,369] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 11:51:32,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:51:33,312] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 11:51:49,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 11:51:50,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 11:51:51,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 11:51:52,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 11:52:02,087] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 11:52:02,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:52:03,473] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 11:52:18,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 11:52:19,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 11:52:20,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 11:52:20,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 11:52:30,677] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 11:56:43,806] INFO : Step 1100/50000 rank2 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1974 | Classification_binary: 0.4089 | Regression_cos_phi: 0.1948 | Regression_energy: 0.3246 | Regression_eta: 0.2222 | Regression_pt: 0.3314 | Regression_sin_phi: 0.2272 | LR: 9.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9222s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank2 [2026-10-28 12:00:30,892] INFO : Step 1200/50000 rank2 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2404 | Classification_binary: 0.4764 | Regression_cos_phi: 0.3083 | Regression_energy: 0.2720 | Regression_eta: 0.2543 | Regression_pt: 0.2747 | Regression_sin_phi: 0.2666 | LR: 9.99e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4001s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank2 [2026-10-28 12:04:26,454] INFO : Step 1300/50000 rank2 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2222 | Classification_binary: 0.4628 | Regression_cos_phi: 0.2773 | Regression_energy: 0.2673 | Regression_eta: 0.2819 | Regression_pt: 0.2744 | Regression_sin_phi: 0.2764 | LR: 9.98e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1276s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank2 [2026-10-28 12:08:10,852] INFO : Step 1400/50000 rank2 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1922 | Classification_binary: 0.3598 | Regression_cos_phi: 0.1885 | Regression_energy: 0.2902 | Regression_eta: 0.2294 | Regression_pt: 0.2989 | Regression_sin_phi: 0.2892 | LR: 9.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7544s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank2 [2026-10-28 12:11:57,637] INFO : Step 1500/50000 rank2 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1927 | Classification_binary: 0.4517 | Regression_cos_phi: 0.2766 | Regression_energy: 0.2592 | Regression_eta: 0.2219 | Regression_pt: 0.2750 | Regression_sin_phi: 0.2647 | LR: 9.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5801s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank2 [2026-10-28 12:15:43,592] INFO : Step 1600/50000 rank2 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1932 | Classification_binary: 0.4136 | Regression_cos_phi: 0.1903 | Regression_energy: 0.2378 | Regression_eta: 0.1872 | Regression_pt: 0.2459 | Regression_sin_phi: 0.2109 | LR: 9.98e-04 | DataLoad Time: 0.0252s | Model Forward Time: 4.0353s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank2 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank2 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1651 | Classification_binary: 0.3827 | Regression_cos_phi: 0.2623 | Regression_energy: 0.2420 | Regression_eta: 0.1878 | Regression_pt: 0.2537 | Regression_sin_phi: 0.1916 | LR: 9.97e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.3921s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank2 [2026-10-28 12:23:17,460] INFO : Step 1800/50000 rank2 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2325 | Classification_binary: 0.3898 | Regression_cos_phi: 0.1999 | Regression_energy: 0.2571 | Regression_eta: 0.2001 | Regression_pt: 0.2692 | Regression_sin_phi: 0.2153 | LR: 9.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9413s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank2 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank2 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2242 | Classification_binary: 0.4406 | Regression_cos_phi: 0.2682 | Regression_energy: 0.2710 | Regression_eta: 0.2012 | Regression_pt: 0.2840 | Regression_sin_phi: 0.2121 | LR: 9.97e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5724s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank2 [2026-10-28 12:30:59,046] INFO : Step 2000/50000 rank2 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.3797 | Regression_cos_phi: 0.2377 | Regression_energy: 0.2107 | Regression_eta: 0.1996 | Regression_pt: 0.2207 | Regression_sin_phi: 0.2503 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1435s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank2 [2026-10-28 12:31:00,063] INFO : Running validation on rank2 +rank2 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 12:31:47,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:31:48,296] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 12:32:13,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:32:14,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:32:15,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:32:16,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:32:26,568] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:32:27,680] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 12:32:43,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:32:44,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:32:45,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:32:46,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:32:56,537] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 12:32:57,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:32:57,641] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 12:33:13,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:33:13,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:33:14,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:33:15,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:33:25,109] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 12:37:32,605] INFO : Step 2100/50000 rank2 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1603 | Classification_binary: 0.3331 | Regression_cos_phi: 0.1393 | Regression_energy: 0.2378 | Regression_eta: 0.1583 | Regression_pt: 0.2476 | Regression_sin_phi: 0.1138 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7728s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank2 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank2 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2257 | Classification_binary: 0.4168 | Regression_cos_phi: 0.2728 | Regression_energy: 0.2605 | Regression_eta: 0.2412 | Regression_pt: 0.2740 | Regression_sin_phi: 0.3190 | LR: 9.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7056s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank2 [2026-10-28 12:45:14,507] INFO : Step 2300/50000 rank2 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2225 | Classification_binary: 0.4552 | Regression_cos_phi: 0.2236 | Regression_energy: 0.2514 | Regression_eta: 0.1939 | Regression_pt: 0.2633 | Regression_sin_phi: 0.2182 | LR: 9.95e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank2 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank2 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1750 | Classification_binary: 0.3248 | Regression_cos_phi: 0.2122 | Regression_energy: 0.1972 | Regression_eta: 0.1797 | Regression_pt: 0.2042 | Regression_sin_phi: 0.1258 | LR: 9.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4488s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank2 [2026-10-28 12:52:58,132] INFO : Step 2500/50000 rank2 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2222 | Classification_binary: 0.4365 | Regression_cos_phi: 0.2479 | Regression_energy: 0.2380 | Regression_eta: 0.2078 | Regression_pt: 0.2511 | Regression_sin_phi: 0.2354 | LR: 9.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4005s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank2 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank2 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2123 | Classification_binary: 0.4853 | Regression_cos_phi: 0.2151 | Regression_energy: 0.2480 | Regression_eta: 0.2030 | Regression_pt: 0.2630 | Regression_sin_phi: 0.2439 | LR: 9.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2537s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank2 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank2 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1589 | Classification_binary: 0.3450 | Regression_cos_phi: 0.1949 | Regression_energy: 0.2619 | Regression_eta: 0.2234 | Regression_pt: 0.2727 | Regression_sin_phi: 0.1741 | LR: 9.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0889s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank2 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank2 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2405 | Classification_binary: 0.3817 | Regression_cos_phi: 0.2642 | Regression_energy: 0.2206 | Regression_eta: 0.2306 | Regression_pt: 0.2377 | Regression_sin_phi: 0.3253 | LR: 9.93e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1941s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank2 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank2 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2210 | Classification_binary: 0.4642 | Regression_cos_phi: 0.3005 | Regression_energy: 0.2412 | Regression_eta: 0.2587 | Regression_pt: 0.2527 | Regression_sin_phi: 0.2860 | LR: 9.93e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1932s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank2 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank2 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.4043 | Regression_cos_phi: 0.1847 | Regression_energy: 0.2146 | Regression_eta: 0.1738 | Regression_pt: 0.2209 | Regression_sin_phi: 0.1813 | LR: 9.92e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.5272s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank2 [2026-10-28 13:12:10,495] INFO : Running validation on rank2 +rank2 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:13:02,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:13:03,432] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 13:13:30,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:13:31,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:13:32,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:13:33,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:13:43,045] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:13:44,110] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 13:14:00,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:14:01,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:14:02,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:14:03,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:14:13,200] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:14:14,291] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 13:14:29,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:14:30,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:14:30,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:14:31,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:14:41,334] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 13:19:02,742] INFO : Step 3100/50000 rank2 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1724 | Classification_binary: 0.3124 | Regression_cos_phi: 0.2044 | Regression_energy: 0.2242 | Regression_eta: 0.1738 | Regression_pt: 0.2414 | Regression_sin_phi: 0.1884 | LR: 9.91e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.5827s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank2 [2026-10-28 13:23:00,673] INFO : Step 3200/50000 rank2 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2239 | Classification_binary: 0.4206 | Regression_cos_phi: 0.2036 | Regression_energy: 0.2540 | Regression_eta: 0.1911 | Regression_pt: 0.2679 | Regression_sin_phi: 0.2324 | LR: 9.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8517s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank2 [2026-10-28 13:26:54,113] INFO : Step 3300/50000 rank2 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1876 | Classification_binary: 0.4211 | Regression_cos_phi: 0.2161 | Regression_energy: 0.2267 | Regression_eta: 0.1946 | Regression_pt: 0.2388 | Regression_sin_phi: 0.2560 | LR: 9.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7312s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank2 [2026-10-28 13:30:48,598] INFO : Step 3400/50000 rank2 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1475 | Classification_binary: 0.2933 | Regression_cos_phi: 0.1569 | Regression_energy: 0.2253 | Regression_eta: 0.1241 | Regression_pt: 0.2330 | Regression_sin_phi: 0.1662 | LR: 9.90e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.0812s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank2 [2026-10-28 13:34:47,771] INFO : Step 3500/50000 rank2 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1781 | Classification_binary: 0.3744 | Regression_cos_phi: 0.1998 | Regression_energy: 0.2230 | Regression_eta: 0.1677 | Regression_pt: 0.2335 | Regression_sin_phi: 0.2206 | LR: 9.89e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6686s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank2 [2026-10-28 13:38:46,123] INFO : Step 3600/50000 rank2 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 0.4534 | Regression_cos_phi: 0.2733 | Regression_energy: 0.1988 | Regression_eta: 0.2124 | Regression_pt: 0.2092 | Regression_sin_phi: 0.2562 | LR: 9.89e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.1031s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank2 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank2 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1587 | Classification_binary: 0.3807 | Regression_cos_phi: 0.2476 | Regression_energy: 0.2196 | Regression_eta: 0.2114 | Regression_pt: 0.2266 | Regression_sin_phi: 0.2468 | LR: 9.88e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2041s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank2 [2026-10-28 13:46:34,311] INFO : Step 3800/50000 rank2 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 0.3390 | Regression_cos_phi: 0.1753 | Regression_energy: 0.2240 | Regression_eta: 0.1831 | Regression_pt: 0.2385 | Regression_sin_phi: 0.2106 | LR: 9.87e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.9862s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank2 [2026-10-28 13:50:23,157] INFO : Step 3900/50000 rank2 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1575 | Classification_binary: 0.4351 | Regression_cos_phi: 0.2197 | Regression_energy: 0.1980 | Regression_eta: 0.1778 | Regression_pt: 0.2056 | Regression_sin_phi: 0.2126 | LR: 9.87e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2433s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank2 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank2 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1671 | Classification_binary: 0.3826 | Regression_cos_phi: 0.2121 | Regression_energy: 0.1985 | Regression_eta: 0.1436 | Regression_pt: 0.2056 | Regression_sin_phi: 0.2308 | LR: 9.86e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5786s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank2 [2026-10-28 13:54:12,844] INFO : Running validation on rank2 +rank2 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:55:03,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 13:55:33,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:55:34,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:55:35,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:55:35,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:55:46,007] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:55:46,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 13:56:03,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:56:04,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:56:05,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:56:06,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:56:16,424] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:56:16,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:56:16,986] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 13:56:32,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:56:32,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:56:33,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:56:34,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:56:44,186] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank2 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1277 | Classification_binary: 0.2941 | Regression_cos_phi: 0.1627 | Regression_energy: 0.2135 | Regression_eta: 0.1662 | Regression_pt: 0.2219 | Regression_sin_phi: 0.2635 | LR: 9.85e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9534s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank2 [2026-10-28 14:04:50,728] INFO : Step 4200/50000 rank2 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1988 | Classification_binary: 0.3789 | Regression_cos_phi: 0.2098 | Regression_energy: 0.2383 | Regression_eta: 0.1976 | Regression_pt: 0.2469 | Regression_sin_phi: 0.2833 | LR: 9.84e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.4847s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank2 [2026-10-28 14:08:42,270] INFO : Step 4300/50000 rank2 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1504 | Classification_binary: 0.4188 | Regression_cos_phi: 0.2202 | Regression_energy: 0.2169 | Regression_eta: 0.1750 | Regression_pt: 0.2282 | Regression_sin_phi: 0.2288 | LR: 9.84e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1049s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank2 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank2 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1359 | Classification_binary: 0.3309 | Regression_cos_phi: 0.1649 | Regression_energy: 0.2173 | Regression_eta: 0.1702 | Regression_pt: 0.2246 | Regression_sin_phi: 0.2308 | LR: 9.83e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.7771s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank2 [2026-10-28 14:16:26,827] INFO : Step 4500/50000 rank2 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1382 | Classification_binary: 0.3302 | Regression_cos_phi: 0.1399 | Regression_energy: 0.2034 | Regression_eta: 0.1434 | Regression_pt: 0.2141 | Regression_sin_phi: 0.1419 | LR: 9.82e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.4663s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank2 [2026-10-28 14:20:31,826] INFO : Step 4600/50000 rank2 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1762 | Classification_binary: 0.3717 | Regression_cos_phi: 0.2057 | Regression_energy: 0.1893 | Regression_eta: 0.1906 | Regression_pt: 0.1976 | Regression_sin_phi: 0.1979 | LR: 9.81e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.1359s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank2 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank2 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.3040 | Regression_cos_phi: 0.1121 | Regression_energy: 0.1943 | Regression_eta: 0.1645 | Regression_pt: 0.1987 | Regression_sin_phi: 0.1437 | LR: 9.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3025s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank2 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank2 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1744 | Classification_binary: 0.3035 | Regression_cos_phi: 0.2588 | Regression_energy: 0.1905 | Regression_eta: 0.1826 | Regression_pt: 0.2004 | Regression_sin_phi: 0.2203 | LR: 9.80e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9658s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank2 [2026-10-28 14:32:01,139] INFO : Step 4900/50000 rank2 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1678 | Classification_binary: 0.3907 | Regression_cos_phi: 0.2017 | Regression_energy: 0.1954 | Regression_eta: 0.1719 | Regression_pt: 0.2060 | Regression_sin_phi: 0.1854 | LR: 9.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8453s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank2 [2026-10-28 14:35:52,795] INFO : Step 5000/50000 rank2 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1472 | Classification_binary: 0.3718 | Regression_cos_phi: 0.2089 | Regression_energy: 0.2081 | Regression_eta: 0.1923 | Regression_pt: 0.2137 | Regression_sin_phi: 0.2192 | LR: 9.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8407s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank2 [2026-10-28 14:35:53,776] INFO : Running validation on rank2 +rank2 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 14:36:42,158] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:36:42,706] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 14:37:08,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 14:37:09,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 14:37:11,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 14:37:12,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 14:37:22,041] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:37:22,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 14:37:38,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 14:37:39,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 14:37:40,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 14:37:41,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 14:37:51,568] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 14:37:52,214] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:37:52,791] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 14:38:07,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 14:38:08,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 14:38:09,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 14:38:09,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 14:38:19,967] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank2 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 0.2953 | Regression_cos_phi: 0.1740 | Regression_energy: 0.2106 | Regression_eta: 0.1372 | Regression_pt: 0.2185 | Regression_sin_phi: 0.1437 | LR: 9.77e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3387s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank2 [2026-10-28 14:46:36,780] INFO : Step 5200/50000 rank2 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1589 | Classification_binary: 0.3446 | Regression_cos_phi: 0.2009 | Regression_energy: 0.1995 | Regression_eta: 0.1390 | Regression_pt: 0.2068 | Regression_sin_phi: 0.1932 | LR: 9.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9489s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank2 [2026-10-28 14:50:33,436] INFO : Step 5300/50000 rank2 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1398 | Classification_binary: 0.3885 | Regression_cos_phi: 0.1650 | Regression_energy: 0.1793 | Regression_eta: 0.1640 | Regression_pt: 0.1873 | Regression_sin_phi: 0.1735 | LR: 9.75e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.1870s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank2 [2026-10-28 14:54:36,491] INFO : Step 5400/50000 rank2 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1311 | Classification_binary: 0.3379 | Regression_cos_phi: 0.1601 | Regression_energy: 0.1742 | Regression_eta: 0.1434 | Regression_pt: 0.1805 | Regression_sin_phi: 0.2197 | LR: 9.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.4926s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank2 [2026-10-28 14:58:42,457] INFO : Step 5500/50000 rank2 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1737 | Classification_binary: 0.3455 | Regression_cos_phi: 0.1928 | Regression_energy: 0.1881 | Regression_eta: 0.1559 | Regression_pt: 0.1965 | Regression_sin_phi: 0.1777 | LR: 9.73e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5920s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank2 [2026-10-28 15:02:44,102] INFO : Step 5600/50000 rank2 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1593 | Classification_binary: 0.3675 | Regression_cos_phi: 0.2166 | Regression_energy: 0.1893 | Regression_eta: 0.1816 | Regression_pt: 0.1985 | Regression_sin_phi: 0.2005 | LR: 9.72e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6402s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank2 [2026-10-28 15:06:32,821] INFO : Step 5700/50000 rank2 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1293 | Classification_binary: 0.3124 | Regression_cos_phi: 0.1124 | Regression_energy: 0.1763 | Regression_eta: 0.1588 | Regression_pt: 0.1832 | Regression_sin_phi: 0.1484 | LR: 9.71e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.1508s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank2 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank2 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.2959 | Regression_cos_phi: 0.2216 | Regression_energy: 0.1838 | Regression_eta: 0.1476 | Regression_pt: 0.1894 | Regression_sin_phi: 0.2016 | LR: 9.70e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5377s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank2 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank2 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1691 | Classification_binary: 0.3807 | Regression_cos_phi: 0.2065 | Regression_energy: 0.2009 | Regression_eta: 0.1794 | Regression_pt: 0.2138 | Regression_sin_phi: 0.2626 | LR: 9.69e-04 | DataLoad Time: 0.0210s | Model Forward Time: 6.0715s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank2 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank2 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1261 | Classification_binary: 0.3541 | Regression_cos_phi: 0.1607 | Regression_energy: 0.2027 | Regression_eta: 0.1435 | Regression_pt: 0.2077 | Regression_sin_phi: 0.1722 | LR: 9.68e-04 | DataLoad Time: 0.0139s | Model Forward Time: 6.3964s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank2 [2026-10-28 15:18:46,644] INFO : Running validation on rank2 +rank2 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:19:35,272] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 15:19:56,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 15:19:57,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 15:19:59,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 15:20:01,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 15:20:11,172] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:20:11,685] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 15:20:27,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 15:20:28,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 15:20:29,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 15:20:30,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 15:20:40,121] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 15:20:40,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:20:41,482] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 15:20:56,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 15:20:57,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 15:20:58,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 15:20:58,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 15:21:08,914] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 15:25:37,243] INFO : Step 6100/50000 rank2 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1425 | Classification_binary: 0.3450 | Regression_cos_phi: 0.2073 | Regression_energy: 0.1714 | Regression_eta: 0.1810 | Regression_pt: 0.1784 | Regression_sin_phi: 0.1867 | LR: 9.67e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2792s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank2 [2026-10-28 15:29:27,138] INFO : Step 6200/50000 rank2 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1386 | Classification_binary: 0.2832 | Regression_cos_phi: 0.1480 | Regression_energy: 0.1927 | Regression_eta: 0.1350 | Regression_pt: 0.2037 | Regression_sin_phi: 0.2040 | LR: 9.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.3978s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank2 [2026-10-28 15:33:16,838] INFO : Step 6300/50000 rank2 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1539 | Classification_binary: 0.3788 | Regression_cos_phi: 0.1607 | Regression_energy: 0.1867 | Regression_eta: 0.1450 | Regression_pt: 0.1980 | Regression_sin_phi: 0.1321 | LR: 9.65e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2465s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank2 [2026-10-28 15:37:09,469] INFO : Step 6400/50000 rank2 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1197 | Classification_binary: 0.2923 | Regression_cos_phi: 0.1864 | Regression_energy: 0.1824 | Regression_eta: 0.1313 | Regression_pt: 0.1940 | Regression_sin_phi: 0.1684 | LR: 9.64e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9973s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank2 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank2 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2901 | Regression_cos_phi: 0.1457 | Regression_energy: 0.1884 | Regression_eta: 0.1401 | Regression_pt: 0.2026 | Regression_sin_phi: 0.1805 | LR: 9.63e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6706s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank2 [2026-10-28 15:45:09,041] INFO : Step 6600/50000 rank2 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 0.3502 | Regression_cos_phi: 0.2047 | Regression_energy: 0.1815 | Regression_eta: 0.1484 | Regression_pt: 0.1895 | Regression_sin_phi: 0.1592 | LR: 9.62e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2531s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank2 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank2 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.3181 | Regression_cos_phi: 0.1671 | Regression_energy: 0.1796 | Regression_eta: 0.1271 | Regression_pt: 0.1876 | Regression_sin_phi: 0.1772 | LR: 9.61e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.0302s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank2 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank2 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1180 | Classification_binary: 0.2648 | Regression_cos_phi: 0.1452 | Regression_energy: 0.1685 | Regression_eta: 0.1289 | Regression_pt: 0.1751 | Regression_sin_phi: 0.1493 | LR: 9.60e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8866s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank2 [2026-10-28 15:56:38,729] INFO : Step 6900/50000 rank2 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1664 | Classification_binary: 0.3384 | Regression_cos_phi: 0.2121 | Regression_energy: 0.1649 | Regression_eta: 0.1885 | Regression_pt: 0.1760 | Regression_sin_phi: 0.2263 | LR: 9.58e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.8825s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank2 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank2 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1478 | Classification_binary: 0.3615 | Regression_cos_phi: 0.2039 | Regression_energy: 0.1759 | Regression_eta: 0.1598 | Regression_pt: 0.1830 | Regression_sin_phi: 0.2003 | LR: 9.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4728s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank2 [2026-10-28 16:00:38,672] INFO : Running validation on rank2 +rank2 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:01:32,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:01:33,504] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 16:02:00,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:02:01,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:02:02,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:02:02,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:02:13,005] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:02:14,184] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 16:02:30,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:02:31,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:02:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:02:32,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:02:42,981] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:02:44,279] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 16:02:59,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:03:00,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:03:01,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:03:02,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:03:12,026] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank2 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1241 | Classification_binary: 0.2903 | Regression_cos_phi: 0.2492 | Regression_energy: 0.1452 | Regression_eta: 0.1954 | Regression_pt: 0.1566 | Regression_sin_phi: 0.2001 | LR: 9.56e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5609s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank2 [2026-10-28 16:11:41,309] INFO : Step 7200/50000 rank2 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.2632 | Regression_cos_phi: 0.1509 | Regression_energy: 0.1761 | Regression_eta: 0.0968 | Regression_pt: 0.1840 | Regression_sin_phi: 0.1250 | LR: 9.55e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9558s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank2 [2026-10-28 16:15:45,432] INFO : Step 7300/50000 rank2 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3937 | Regression_cos_phi: 0.1998 | Regression_energy: 0.1587 | Regression_eta: 0.1655 | Regression_pt: 0.1659 | Regression_sin_phi: 0.2349 | LR: 9.53e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.8994s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank2 [2026-10-28 16:19:47,791] INFO : Step 7400/50000 rank2 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1257 | Classification_binary: 0.3002 | Regression_cos_phi: 0.1365 | Regression_energy: 0.2083 | Regression_eta: 0.1275 | Regression_pt: 0.2177 | Regression_sin_phi: 0.1466 | LR: 9.52e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6030s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank2 [2026-10-28 16:23:56,337] INFO : Step 7500/50000 rank2 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1179 | Classification_binary: 0.2745 | Regression_cos_phi: 0.1864 | Regression_energy: 0.1732 | Regression_eta: 0.1403 | Regression_pt: 0.1805 | Regression_sin_phi: 0.1754 | LR: 9.51e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7930s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank2 [2026-10-28 16:27:58,400] INFO : Step 7600/50000 rank2 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1780 | Classification_binary: 0.3685 | Regression_cos_phi: 0.1866 | Regression_energy: 0.1894 | Regression_eta: 0.1890 | Regression_pt: 0.1991 | Regression_sin_phi: 0.1741 | LR: 9.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7007s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank2 [2026-10-28 16:32:00,294] INFO : Step 7700/50000 rank2 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1301 | Classification_binary: 0.3691 | Regression_cos_phi: 0.1904 | Regression_energy: 0.1690 | Regression_eta: 0.1553 | Regression_pt: 0.1764 | Regression_sin_phi: 0.1885 | LR: 9.48e-04 | DataLoad Time: 0.0126s | Model Forward Time: 4.9846s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank2 [2026-10-28 16:35:55,393] INFO : Step 7800/50000 rank2 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1319 | Classification_binary: 0.2824 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1621 | Regression_eta: 0.1471 | Regression_pt: 0.1681 | Regression_sin_phi: 0.1727 | LR: 9.47e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2121s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank2 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank2 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1325 | Classification_binary: 0.3165 | Regression_cos_phi: 0.1701 | Regression_energy: 0.1695 | Regression_eta: 0.1529 | Regression_pt: 0.1792 | Regression_sin_phi: 0.1689 | LR: 9.46e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3410s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank2 [2026-10-28 16:44:12,791] INFO : Step 8000/50000 rank2 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1265 | Classification_binary: 0.3781 | Regression_cos_phi: 0.1984 | Regression_energy: 0.1708 | Regression_eta: 0.1504 | Regression_pt: 0.1795 | Regression_sin_phi: 0.1842 | LR: 9.44e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.2430s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank2 [2026-10-28 16:44:13,936] INFO : Running validation on rank2 +rank2 [2026-10-28 16:44:13,936] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:45:04,006] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:45:04,562] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 16:45:30,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:45:31,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:45:32,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:45:33,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:45:43,934] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:45:45,475] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 16:46:01,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:46:02,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:46:03,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:46:04,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:46:14,218] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:46:15,423] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 16:46:30,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:46:31,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:46:32,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:46:32,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:46:42,706] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank2 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0972 | Classification_binary: 0.2685 | Regression_cos_phi: 0.1268 | Regression_energy: 0.1560 | Regression_eta: 0.1064 | Regression_pt: 0.1606 | Regression_sin_phi: 0.1133 | LR: 9.43e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7753s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank2 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank2 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.2636 | Regression_cos_phi: 0.1496 | Regression_energy: 0.1460 | Regression_eta: 0.1196 | Regression_pt: 0.1555 | Regression_sin_phi: 0.1751 | LR: 9.42e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1556s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank2 [2026-10-28 16:59:31,718] INFO : Step 8300/50000 rank2 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1113 | Classification_binary: 0.3426 | Regression_cos_phi: 0.1707 | Regression_energy: 0.1632 | Regression_eta: 0.1546 | Regression_pt: 0.1706 | Regression_sin_phi: 0.1547 | LR: 9.40e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.9203s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank2 [2026-10-28 17:03:26,316] INFO : Step 8400/50000 rank2 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1283 | Classification_binary: 0.3058 | Regression_cos_phi: 0.1854 | Regression_energy: 0.1600 | Regression_eta: 0.1319 | Regression_pt: 0.1652 | Regression_sin_phi: 0.2311 | LR: 9.39e-04 | DataLoad Time: 0.0217s | Model Forward Time: 6.0460s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank2 [2026-10-28 17:07:18,317] INFO : Step 8500/50000 rank2 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1084 | Classification_binary: 0.2514 | Regression_cos_phi: 0.1604 | Regression_energy: 0.1391 | Regression_eta: 0.1476 | Regression_pt: 0.1447 | Regression_sin_phi: 0.1727 | LR: 9.37e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.4973s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank2 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank2 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1399 | Classification_binary: 0.2886 | Regression_cos_phi: 0.1846 | Regression_energy: 0.1538 | Regression_eta: 0.1487 | Regression_pt: 0.1608 | Regression_sin_phi: 0.1548 | LR: 9.36e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1705s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank2 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank2 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.3163 | Regression_cos_phi: 0.1499 | Regression_energy: 0.1548 | Regression_eta: 0.1329 | Regression_pt: 0.1603 | Regression_sin_phi: 0.1294 | LR: 9.34e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.2158s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank2 [2026-10-28 17:19:40,248] INFO : Step 8800/50000 rank2 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2428 | Regression_cos_phi: 0.1119 | Regression_energy: 0.1579 | Regression_eta: 0.1167 | Regression_pt: 0.1631 | Regression_sin_phi: 0.1378 | LR: 9.33e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.4435s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank2 [2026-10-28 17:23:38,393] INFO : Step 8900/50000 rank2 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1363 | Classification_binary: 0.3603 | Regression_cos_phi: 0.1898 | Regression_energy: 0.1664 | Regression_eta: 0.1579 | Regression_pt: 0.1732 | Regression_sin_phi: 0.1929 | LR: 9.31e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6555s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank2 [2026-10-28 17:27:33,249] INFO : Step 9000/50000 rank2 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1334 | Classification_binary: 0.3845 | Regression_cos_phi: 0.1685 | Regression_energy: 0.1596 | Regression_eta: 0.1480 | Regression_pt: 0.1663 | Regression_sin_phi: 0.2091 | LR: 9.30e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8297s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank2 [2026-10-28 17:27:34,333] INFO : Running validation on rank2 +rank2 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 17:28:24,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:28:25,265] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 17:28:52,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 17:28:53,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 17:28:54,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 17:28:55,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 17:29:05,058] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:29:06,070] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 17:29:22,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 17:29:23,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 17:29:24,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 17:29:25,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 17:29:35,879] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 17:29:37,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:29:37,947] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 17:29:53,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 17:29:53,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 17:29:54,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 17:29:55,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 17:30:05,217] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 17:34:23,898] INFO : Step 9100/50000 rank2 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2973 | Regression_cos_phi: 0.1063 | Regression_energy: 0.1537 | Regression_eta: 0.0945 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0945 | LR: 9.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1044s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank2 [2026-10-28 17:38:24,814] INFO : Step 9200/50000 rank2 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2170 | Regression_cos_phi: 0.1126 | Regression_energy: 0.1338 | Regression_eta: 0.0935 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0869 | LR: 9.27e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3251s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank2 [2026-10-28 17:42:44,650] INFO : Step 9300/50000 rank2 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1076 | Classification_binary: 0.2977 | Regression_cos_phi: 0.1759 | Regression_energy: 0.1721 | Regression_eta: 0.1542 | Regression_pt: 0.1810 | Regression_sin_phi: 0.1908 | LR: 9.25e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.8249s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank2 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank2 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2841 | Regression_cos_phi: 0.1592 | Regression_energy: 0.1364 | Regression_eta: 0.1162 | Regression_pt: 0.1430 | Regression_sin_phi: 0.1368 | LR: 9.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4157s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank2 [2026-10-28 17:51:11,694] INFO : Step 9500/50000 rank2 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0939 | Classification_binary: 0.2379 | Regression_cos_phi: 0.1531 | Regression_energy: 0.1460 | Regression_eta: 0.1181 | Regression_pt: 0.1505 | Regression_sin_phi: 0.1746 | LR: 9.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.2536s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank2 [2026-10-28 17:55:19,473] INFO : Step 9600/50000 rank2 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1059 | Classification_binary: 0.2754 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1465 | Regression_eta: 0.1121 | Regression_pt: 0.1531 | Regression_sin_phi: 0.1417 | LR: 9.21e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.8384s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank2 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank2 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.3344 | Regression_cos_phi: 0.1471 | Regression_energy: 0.1392 | Regression_eta: 0.1341 | Regression_pt: 0.1449 | Regression_sin_phi: 0.1435 | LR: 9.19e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9955s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank2 [2026-10-28 18:03:29,394] INFO : Step 9800/50000 rank2 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2345 | Regression_cos_phi: 0.1295 | Regression_energy: 0.1462 | Regression_eta: 0.0905 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0867 | LR: 9.17e-04 | DataLoad Time: 0.0227s | Model Forward Time: 3.6857s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank2 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank2 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2399 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1407 | Regression_eta: 0.1006 | Regression_pt: 0.1477 | Regression_sin_phi: 0.1175 | LR: 9.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1749s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank2 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank2 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.3292 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1278 | Regression_eta: 0.1003 | Regression_pt: 0.1357 | Regression_sin_phi: 0.1396 | LR: 9.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1264s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank2 [2026-10-28 18:11:36,666] INFO : Running validation on rank2 +rank2 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:12:31,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:12:31,988] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 18:12:58,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:12:59,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:13:00,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:13:01,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:13:11,202] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:13:12,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:13:12,682] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 18:13:28,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:13:30,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:13:31,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:13:32,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:13:42,167] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 18:13:48,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:13:49,182] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 18:14:04,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:14:05,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:14:06,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:14:06,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:14:16,809] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 18:18:32,574] INFO : Step 10100/50000 rank2 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.3260 | Regression_cos_phi: 0.1275 | Regression_energy: 0.1351 | Regression_eta: 0.1296 | Regression_pt: 0.1416 | Regression_sin_phi: 0.1563 | LR: 9.12e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2363s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank2 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank2 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0891 | Classification_binary: 0.2374 | Regression_cos_phi: 0.1344 | Regression_energy: 0.1309 | Regression_eta: 0.1009 | Regression_pt: 0.1371 | Regression_sin_phi: 0.1237 | LR: 9.11e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5682s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank2 [2026-10-28 18:26:52,297] INFO : Step 10300/50000 rank2 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1165 | Classification_binary: 0.3472 | Regression_cos_phi: 0.1559 | Regression_energy: 0.1233 | Regression_eta: 0.1221 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1344 | LR: 9.09e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.3481s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank2 [2026-10-28 18:31:09,157] INFO : Step 10400/50000 rank2 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.3218 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1458 | Regression_eta: 0.1265 | Regression_pt: 0.1517 | Regression_sin_phi: 0.1492 | LR: 9.07e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9067s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank2 [2026-10-28 18:35:25,946] INFO : Step 10500/50000 rank2 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2154 | Regression_cos_phi: 0.1022 | Regression_energy: 0.1437 | Regression_eta: 0.0989 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1039 | LR: 9.06e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8296s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank2 [2026-10-28 18:39:26,895] INFO : Step 10600/50000 rank2 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2951 | Regression_cos_phi: 0.1386 | Regression_energy: 0.1213 | Regression_eta: 0.0898 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1231 | LR: 9.04e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2193s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank2 [2026-10-28 18:43:27,488] INFO : Step 10700/50000 rank2 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.3051 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1362 | Regression_eta: 0.1029 | Regression_pt: 0.1438 | Regression_sin_phi: 0.1451 | LR: 9.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2761s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank2 [2026-10-28 18:47:34,479] INFO : Step 10800/50000 rank2 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2460 | Regression_cos_phi: 0.0794 | Regression_energy: 0.1246 | Regression_eta: 0.0762 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0679 | LR: 9.00e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.3736s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank2 [2026-10-28 18:51:51,744] INFO : Step 10900/50000 rank2 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2572 | Regression_cos_phi: 0.1027 | Regression_energy: 0.1124 | Regression_eta: 0.0635 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0946 | LR: 8.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6031s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank2 [2026-10-28 18:55:59,247] INFO : Step 11000/50000 rank2 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1419 | Regression_energy: 0.1192 | Regression_eta: 0.0990 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1050 | LR: 8.97e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.8634s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank2 [2026-10-28 18:56:00,216] INFO : Running validation on rank2 +rank2 [2026-10-28 18:56:00,216] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:56:54,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:56:55,375] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 18:57:21,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:57:22,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:57:23,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:57:24,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:57:34,869] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:57:35,271] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:57:36,000] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 18:57:52,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:57:53,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:57:54,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:57:55,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:58:05,685] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:58:06,770] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 18:58:22,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:58:22,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:58:23,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:58:24,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:58:34,260] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 19:03:04,100] INFO : Step 11100/50000 rank2 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.2783 | Regression_cos_phi: 0.1160 | Regression_energy: 0.1147 | Regression_eta: 0.0742 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0996 | LR: 8.95e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4763s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank2 [2026-10-28 19:07:10,330] INFO : Step 11200/50000 rank2 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2057 | Regression_cos_phi: 0.1022 | Regression_energy: 0.1388 | Regression_eta: 0.1238 | Regression_pt: 0.1501 | Regression_sin_phi: 0.1087 | LR: 8.93e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8094s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank2 [2026-10-28 19:11:04,858] INFO : Step 11300/50000 rank2 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.3018 | Regression_cos_phi: 0.1144 | Regression_energy: 0.1181 | Regression_eta: 0.0910 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1045 | LR: 8.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2488s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank2 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank2 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.3267 | Regression_cos_phi: 0.1385 | Regression_energy: 0.1185 | Regression_eta: 0.1234 | Regression_pt: 0.1241 | Regression_sin_phi: 0.1108 | LR: 8.89e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0024s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank2 [2026-10-28 19:18:58,942] INFO : Step 11500/50000 rank2 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.2356 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1045 | Regression_eta: 0.1094 | Regression_pt: 0.1113 | Regression_sin_phi: 0.1040 | LR: 8.88e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.4525s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank2 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank2 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1021 | Classification_binary: 0.2953 | Regression_cos_phi: 0.1255 | Regression_energy: 0.1237 | Regression_eta: 0.1094 | Regression_pt: 0.1271 | Regression_sin_phi: 0.1096 | LR: 8.86e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8310s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank2 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank2 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1179 | Regression_eta: 0.0877 | Regression_pt: 0.1224 | Regression_sin_phi: 0.1473 | LR: 8.84e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1779s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank2 [2026-10-28 19:30:54,032] INFO : Step 11800/50000 rank2 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2658 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1182 | Regression_eta: 0.0921 | Regression_pt: 0.1241 | Regression_sin_phi: 0.1091 | LR: 8.82e-04 | DataLoad Time: 0.0253s | Model Forward Time: 6.0731s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank2 [2026-10-28 19:34:55,961] INFO : Step 11900/50000 rank2 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2511 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0996 | Regression_eta: 0.0695 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0887 | LR: 8.80e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.9520s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank2 [2026-10-28 19:39:04,012] INFO : Step 12000/50000 rank2 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.3049 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1243 | Regression_eta: 0.1024 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0987 | LR: 8.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3562s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank2 [2026-10-28 19:39:04,976] INFO : Running validation on rank2 +rank2 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 19:39:55,048] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:39:55,583] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 19:40:21,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 19:40:22,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 19:40:23,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 19:40:24,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 19:40:34,453] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:40:34,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:40:35,409] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 19:40:51,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 19:40:52,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 19:40:53,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 19:40:54,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 19:41:04,253] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:41:05,105] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:41:05,610] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 19:41:20,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 19:41:21,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 19:41:22,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 19:41:22,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 19:41:32,861] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 19:45:57,645] INFO : Step 12100/50000 rank2 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2920 | Regression_cos_phi: 0.1175 | Regression_energy: 0.1087 | Regression_eta: 0.1096 | Regression_pt: 0.1131 | Regression_sin_phi: 0.1354 | LR: 8.76e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8603s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank2 [2026-10-28 19:50:02,390] INFO : Step 12200/50000 rank2 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2109 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0993 | Regression_eta: 0.0800 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0794 | LR: 8.74e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9610s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank2 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank2 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.2628 | Regression_cos_phi: 0.0669 | Regression_energy: 0.1115 | Regression_eta: 0.0821 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0890 | LR: 8.72e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4607s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank2 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank2 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2975 | Regression_cos_phi: 0.0931 | Regression_energy: 0.1053 | Regression_eta: 0.0735 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0903 | LR: 8.70e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9492s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank2 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank2 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0703 | Regression_energy: 0.1188 | Regression_eta: 0.0968 | Regression_pt: 0.1258 | Regression_sin_phi: 0.1165 | LR: 8.68e-04 | DataLoad Time: 0.0242s | Model Forward Time: 4.6926s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank2 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank2 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.2012 | Regression_cos_phi: 0.1270 | Regression_energy: 0.0998 | Regression_eta: 0.1296 | Regression_pt: 0.1012 | Regression_sin_phi: 0.1140 | LR: 8.66e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9519s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank2 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank2 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0852 | Classification_binary: 0.2609 | Regression_cos_phi: 0.1395 | Regression_energy: 0.1229 | Regression_eta: 0.1071 | Regression_pt: 0.1305 | Regression_sin_phi: 0.1288 | LR: 8.64e-04 | DataLoad Time: 0.0105s | Model Forward Time: 5.2335s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank2 [2026-10-28 20:14:52,957] INFO : Step 12800/50000 rank2 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2693 | Regression_cos_phi: 0.1124 | Regression_energy: 0.0947 | Regression_eta: 0.0839 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0861 | LR: 8.62e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.8531s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank2 [2026-10-28 20:19:01,949] INFO : Step 12900/50000 rank2 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0806 | Regression_energy: 0.1066 | Regression_eta: 0.0648 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0916 | LR: 8.60e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5186s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank2 [2026-10-28 20:23:25,507] INFO : Step 13000/50000 rank2 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2991 | Regression_cos_phi: 0.0714 | Regression_energy: 0.1025 | Regression_eta: 0.0707 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0960 | LR: 8.58e-04 | DataLoad Time: 0.0154s | Model Forward Time: 10.3286s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank2 [2026-10-28 20:23:26,439] INFO : Running validation on rank2 +rank2 [2026-10-28 20:23:26,439] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 20:23:26,439] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 20:24:18,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 20:24:45,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 20:24:46,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 20:24:47,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 20:24:48,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 20:24:58,375] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 20:25:04,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:25:04,757] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 20:25:20,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 20:25:21,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 20:25:22,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 20:25:23,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 20:25:33,518] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:25:34,728] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 20:25:50,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 20:25:51,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 20:25:51,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 20:25:52,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 20:26:02,317] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 20:30:28,065] INFO : Step 13100/50000 rank2 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2796 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1013 | Regression_eta: 0.0852 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0963 | LR: 8.56e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0207s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank2 [2026-10-28 20:34:38,794] INFO : Step 13200/50000 rank2 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0857 | Regression_eta: 0.1084 | Regression_pt: 0.0899 | Regression_sin_phi: 0.1306 | LR: 8.54e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0420s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank2 [2026-10-28 20:38:50,933] INFO : Step 13300/50000 rank2 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2744 | Regression_cos_phi: 0.1184 | Regression_energy: 0.0987 | Regression_eta: 0.1089 | Regression_pt: 0.1031 | Regression_sin_phi: 0.1421 | LR: 8.52e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.4738s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank2 [2026-10-28 20:43:04,349] INFO : Step 13400/50000 rank2 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2911 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0939 | Regression_eta: 0.1016 | Regression_pt: 0.0979 | Regression_sin_phi: 0.1019 | LR: 8.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2860s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank2 [2026-10-28 20:47:13,647] INFO : Step 13500/50000 rank2 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2642 | Regression_cos_phi: 0.1177 | Regression_energy: 0.0902 | Regression_eta: 0.0947 | Regression_pt: 0.0927 | Regression_sin_phi: 0.1136 | LR: 8.48e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1196s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank2 [2026-10-28 20:51:26,251] INFO : Step 13600/50000 rank2 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0811 | Regression_eta: 0.0947 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0977 | LR: 8.45e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.8093s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank2 [2026-10-28 20:55:41,370] INFO : Step 13700/50000 rank2 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.3188 | Regression_cos_phi: 0.1151 | Regression_energy: 0.0948 | Regression_eta: 0.0693 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0919 | LR: 8.43e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.7987s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank2 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank2 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2600 | Regression_cos_phi: 0.1276 | Regression_energy: 0.1021 | Regression_eta: 0.0938 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0943 | LR: 8.41e-04 | DataLoad Time: 0.0222s | Model Forward Time: 6.1143s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank2 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank2 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0516 | Regression_energy: 0.1058 | Regression_eta: 0.0671 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0840 | LR: 8.39e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.1586s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank2 [2026-10-28 21:08:24,061] INFO : Step 14000/50000 rank2 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2591 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0973 | Regression_eta: 0.0728 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0723 | LR: 8.37e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.9188s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank2 [2026-10-28 21:08:24,998] INFO : Running validation on rank2 +rank2 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:09:16,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 21:09:43,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:09:44,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:09:45,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:09:45,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:09:55,993] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:09:56,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:09:57,200] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 21:10:13,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:10:14,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:10:15,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:10:16,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:10:26,144] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:10:26,832] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 21:10:42,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:10:43,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:10:43,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:10:44,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:10:54,569] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 21:15:28,842] INFO : Step 14100/50000 rank2 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0890 | Regression_eta: 0.0630 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0826 | LR: 8.35e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5882s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank2 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank2 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.2199 | Regression_cos_phi: 0.1166 | Regression_energy: 0.0800 | Regression_eta: 0.0814 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0603 | LR: 8.32e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4863s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank2 [2026-10-28 21:24:03,129] INFO : Step 14300/50000 rank2 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2332 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0890 | Regression_eta: 0.0766 | Regression_pt: 0.0915 | Regression_sin_phi: 0.1070 | LR: 8.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0240s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank2 [2026-10-28 21:28:09,727] INFO : Step 14400/50000 rank2 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.3086 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0836 | Regression_eta: 0.0728 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0893 | LR: 8.28e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.0270s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank2 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank2 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0984 | Regression_eta: 0.0782 | Regression_pt: 0.1016 | Regression_sin_phi: 0.1212 | LR: 8.26e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.3480s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank2 [2026-10-28 21:36:11,052] INFO : Step 14600/50000 rank2 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2503 | Regression_cos_phi: 0.1143 | Regression_energy: 0.1032 | Regression_eta: 0.0660 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0745 | LR: 8.24e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.9786s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank2 [2026-10-28 21:40:16,608] INFO : Step 14700/50000 rank2 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0886 | Classification_binary: 0.3103 | Regression_cos_phi: 0.1052 | Regression_energy: 0.1055 | Regression_eta: 0.0895 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0950 | LR: 8.21e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1382s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank2 [2026-10-28 21:44:33,910] INFO : Step 14800/50000 rank2 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1958 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0954 | Regression_eta: 0.0931 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0800 | LR: 8.19e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.3516s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank2 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank2 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0827 | Regression_eta: 0.0666 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0511 | LR: 8.17e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.3234s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank2 [2026-10-28 21:52:53,291] INFO : Step 15000/50000 rank2 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0860 | Classification_binary: 0.2490 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0936 | Regression_eta: 0.0579 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0869 | LR: 8.15e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7242s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank2 [2026-10-28 21:52:54,209] INFO : Running validation on rank2 +rank2 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:53:44,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:53:44,573] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 21:54:10,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:54:11,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:54:12,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:54:13,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:54:23,811] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:54:24,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:54:24,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 21:54:40,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:54:41,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:54:42,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:54:43,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:54:53,572] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:54:54,913] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 21:55:10,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:55:11,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:55:11,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:55:12,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:55:22,587] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 21:59:40,818] INFO : Step 15100/50000 rank2 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0828 | Regression_eta: 0.0938 | Regression_pt: 0.0851 | Regression_sin_phi: 0.1012 | LR: 8.12e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.6883s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank2 [2026-10-28 22:03:40,544] INFO : Step 15200/50000 rank2 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.1932 | Regression_cos_phi: 0.1321 | Regression_energy: 0.0944 | Regression_eta: 0.0876 | Regression_pt: 0.0965 | Regression_sin_phi: 0.1449 | LR: 8.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9685s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank2 [2026-10-28 22:07:35,568] INFO : Step 15300/50000 rank2 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.2592 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0826 | Regression_eta: 0.0735 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0806 | LR: 8.08e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8109s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank2 [2026-10-28 22:11:47,343] INFO : Step 15400/50000 rank2 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.3058 | Regression_cos_phi: 0.1417 | Regression_energy: 0.1020 | Regression_eta: 0.1050 | Regression_pt: 0.1080 | Regression_sin_phi: 0.1410 | LR: 8.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1744s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank2 [2026-10-28 22:16:06,325] INFO : Step 15500/50000 rank2 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0964 | Regression_eta: 0.0858 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0816 | LR: 8.03e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9221s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank2 [2026-10-28 22:20:18,325] INFO : Step 15600/50000 rank2 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0769 | Regression_eta: 0.0690 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0629 | LR: 8.01e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.4201s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank2 [2026-10-28 22:24:30,257] INFO : Step 15700/50000 rank2 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.2647 | Regression_cos_phi: 0.1053 | Regression_energy: 0.0783 | Regression_eta: 0.0912 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0748 | LR: 7.98e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.9306s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank2 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank2 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0885 | Regression_eta: 0.0773 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0904 | LR: 7.96e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3024s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank2 [2026-10-28 22:32:57,711] INFO : Step 15900/50000 rank2 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0792 | Regression_eta: 0.0620 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0680 | LR: 7.94e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.3906s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank2 [2026-10-28 22:37:10,741] INFO : Step 16000/50000 rank2 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.2364 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0879 | Regression_eta: 0.0954 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0829 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0608s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank2 [2026-10-28 22:37:11,680] INFO : Running validation on rank2 +rank2 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 22:38:03,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:38:04,032] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 22:38:30,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 22:38:31,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 22:38:32,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 22:38:33,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 22:38:43,282] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:38:43,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:38:44,323] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 22:39:00,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 22:39:01,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 22:39:03,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 22:39:04,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 22:39:14,465] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 22:39:15,560] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:39:16,106] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 22:39:31,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 22:39:32,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 22:39:32,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 22:39:33,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 22:39:43,502] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 22:44:03,866] INFO : Step 16100/50000 rank2 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2356 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0752 | Regression_eta: 0.0699 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0702 | LR: 7.89e-04 | DataLoad Time: 0.0119s | Model Forward Time: 4.7834s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank2 [2026-10-28 22:48:09,286] INFO : Step 16200/50000 rank2 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0913 | Regression_eta: 0.1021 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0721 | LR: 7.86e-04 | DataLoad Time: 0.0226s | Model Forward Time: 3.6642s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank2 [2026-10-28 22:52:16,252] INFO : Step 16300/50000 rank2 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2344 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0863 | Regression_eta: 0.0888 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0997 | LR: 7.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1368s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank2 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank2 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2417 | Regression_cos_phi: 0.1149 | Regression_energy: 0.0741 | Regression_eta: 0.0718 | Regression_pt: 0.0768 | Regression_sin_phi: 0.1107 | LR: 7.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4183s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank2 [2026-10-28 23:00:23,724] INFO : Step 16500/50000 rank2 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0792 | Regression_eta: 0.0832 | Regression_pt: 0.0807 | Regression_sin_phi: 0.1124 | LR: 7.79e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0017s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank2 [2026-10-28 23:04:26,317] INFO : Step 16600/50000 rank2 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0908 | Classification_binary: 0.1892 | Regression_cos_phi: 0.1126 | Regression_energy: 0.0699 | Regression_eta: 0.0631 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0831 | LR: 7.77e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3635s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank2 [2026-10-28 23:08:31,119] INFO : Step 16700/50000 rank2 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2620 | Regression_cos_phi: 0.1342 | Regression_energy: 0.1002 | Regression_eta: 0.1443 | Regression_pt: 0.1043 | Regression_sin_phi: 0.1316 | LR: 7.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0569s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank2 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank2 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2385 | Regression_cos_phi: 0.1161 | Regression_energy: 0.1050 | Regression_eta: 0.0703 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0914 | LR: 7.72e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.0645s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank2 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank2 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.1919 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0826 | Regression_eta: 0.0648 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1142 | LR: 7.69e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.7647s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank2 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank2 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1998 | Regression_cos_phi: 0.1020 | Regression_energy: 0.0992 | Regression_eta: 0.0682 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0826 | LR: 7.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9353s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank2 [2026-10-28 23:20:37,935] INFO : Running validation on rank2 +rank2 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 23:21:28,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:21:29,098] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 23:21:55,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 23:21:57,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 23:22:00,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 23:22:02,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 23:22:12,062] INFO : Time taken to make predictions on device 2 is: 0.32 min +rank2 [2026-10-28 23:22:12,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:22:12,960] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 23:22:29,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 23:22:30,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 23:22:30,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 23:22:31,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 23:22:41,816] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 23:22:42,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:22:42,878] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 23:22:58,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 23:22:59,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 23:22:59,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 23:23:00,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 23:23:10,349] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 23:27:38,316] INFO : Step 17100/50000 rank2 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0706 | Regression_eta: 0.0608 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0627 | LR: 7.64e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1681s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank2 [2026-10-28 23:31:38,672] INFO : Step 17200/50000 rank2 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2111 | Regression_cos_phi: 0.1260 | Regression_energy: 0.0865 | Regression_eta: 0.0835 | Regression_pt: 0.0924 | Regression_sin_phi: 0.1103 | LR: 7.62e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1955s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank2 [2026-10-28 23:35:49,539] INFO : Step 17300/50000 rank2 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0953 | Regression_eta: 0.0667 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0878 | LR: 7.59e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.5112s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank2 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank2 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2536 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0727 | Regression_eta: 0.0818 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0904 | LR: 7.57e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.3515s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank2 [2026-10-28 23:43:53,975] INFO : Step 17500/50000 rank2 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0710 | Regression_eta: 0.0569 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0459 | LR: 7.54e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.9638s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank2 [2026-10-28 23:48:08,875] INFO : Step 17600/50000 rank2 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1989 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0777 | Regression_eta: 0.0650 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0631 | LR: 7.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5571s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank2 [2026-10-28 23:52:17,699] INFO : Step 17700/50000 rank2 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0764 | Regression_eta: 0.0517 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0576 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.8654s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank2 [2026-10-28 23:56:40,920] INFO : Step 17800/50000 rank2 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0802 | Regression_eta: 0.0645 | Regression_pt: 0.0834 | Regression_sin_phi: 0.1000 | LR: 7.47e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.1273s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank2 [2026-10-29 00:00:47,130] INFO : Step 17900/50000 rank2 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0825 | Regression_eta: 0.0495 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0975 | LR: 7.44e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2731s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank2 [2026-10-29 00:04:53,689] INFO : Step 18000/50000 rank2 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0774 | Regression_eta: 0.0515 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0648 | LR: 7.42e-04 | DataLoad Time: 0.0159s | Model Forward Time: 13.5880s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank2 [2026-10-29 00:04:54,634] INFO : Running validation on rank2 +rank2 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:05:43,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:05:43,820] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 00:06:10,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:06:11,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:06:12,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:06:14,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:06:24,105] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 00:06:24,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 00:06:41,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:06:42,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:06:43,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:06:44,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:06:54,252] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 00:07:10,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:07:11,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:07:12,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:07:12,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:07:22,852] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 00:11:38,772] INFO : Step 18100/50000 rank2 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2815 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0804 | Regression_eta: 0.0853 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0926 | LR: 7.39e-04 | DataLoad Time: 0.0257s | Model Forward Time: 4.2911s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank2 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank2 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0746 | Regression_eta: 0.0461 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0633 | LR: 7.36e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1143s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank2 [2026-10-29 00:19:54,086] INFO : Step 18300/50000 rank2 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.1882 | Regression_cos_phi: 0.1031 | Regression_energy: 0.0765 | Regression_eta: 0.0982 | Regression_pt: 0.0805 | Regression_sin_phi: 0.1049 | LR: 7.34e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8057s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank2 [2026-10-29 00:24:07,391] INFO : Step 18400/50000 rank2 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2523 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0786 | Regression_eta: 0.0630 | Regression_pt: 0.0823 | Regression_sin_phi: 0.1025 | LR: 7.31e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4824s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank2 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank2 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0833 | Regression_eta: 0.0774 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0743 | LR: 7.29e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank2 [2026-10-29 00:31:24,472] INFO : Step 18600/50000 rank2 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0812 | Regression_eta: 0.0656 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0598 | LR: 7.26e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1895s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank2 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank2 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.2297 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0856 | Regression_eta: 0.0618 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0922 | LR: 7.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.9497s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank2 [2026-10-29 00:38:49,580] INFO : Step 18800/50000 rank2 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2250 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0708 | Regression_eta: 0.0595 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0582 | LR: 7.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4356s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank2 [2026-10-29 00:42:28,238] INFO : Step 18900/50000 rank2 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0758 | Regression_eta: 0.0715 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0884 | LR: 7.18e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0105s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank2 [2026-10-29 00:46:10,109] INFO : Step 19000/50000 rank2 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0963 | Regression_eta: 0.0995 | Regression_pt: 0.0987 | Regression_sin_phi: 0.1068 | LR: 7.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1520s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank2 [2026-10-29 00:46:11,054] INFO : Running validation on rank2 +rank2 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:46:58,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:46:59,173] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 00:47:25,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:47:26,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:47:27,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:47:28,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:47:38,146] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:47:38,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:47:38,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 00:47:54,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:47:55,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:47:56,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:47:57,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:48:07,274] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:48:09,174] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 00:48:24,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:48:25,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:48:25,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:48:26,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:48:36,537] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank2 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2532 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0993 | Regression_eta: 0.0609 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0680 | LR: 7.13e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5907s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank2 [2026-10-29 00:56:31,490] INFO : Step 19200/50000 rank2 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0855 | Regression_eta: 0.0688 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0883 | LR: 7.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7718s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank2 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank2 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0747 | Regression_eta: 0.0575 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0738 | LR: 7.08e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.6455s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank2 [2026-10-29 01:03:57,848] INFO : Step 19400/50000 rank2 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2797 | Regression_cos_phi: 0.1298 | Regression_energy: 0.0842 | Regression_eta: 0.1039 | Regression_pt: 0.0882 | Regression_sin_phi: 0.1291 | LR: 7.05e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1733s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank2 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank2 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2369 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0858 | Regression_eta: 0.0767 | Regression_pt: 0.0922 | Regression_sin_phi: 0.1037 | LR: 7.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0505s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank2 [2026-10-29 01:11:25,842] INFO : Step 19600/50000 rank2 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2002 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0766 | Regression_eta: 0.0813 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0830 | LR: 7.00e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.9373s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank2 [2026-10-29 01:15:28,502] INFO : Step 19700/50000 rank2 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0844 | Regression_eta: 0.0445 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0782 | LR: 6.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.3547s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank2 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank2 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2289 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0838 | Regression_eta: 0.0578 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0845 | LR: 6.94e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3384s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank2 [2026-10-29 01:23:26,314] INFO : Step 19900/50000 rank2 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0884 | Regression_eta: 0.0837 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0802 | LR: 6.92e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.7475s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank2 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank2 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0889 | Regression_eta: 0.0891 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0801 | LR: 6.89e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8754s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank2 [2026-10-29 01:27:16,848] INFO : Running validation on rank2 +rank2 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 01:28:06,612] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:28:07,181] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 01:28:33,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 01:28:34,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 01:28:36,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 01:28:37,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 01:28:47,138] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 01:28:48,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:28:48,587] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 01:29:04,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 01:29:05,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 01:29:06,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 01:29:07,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 01:29:17,146] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 01:29:18,019] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:29:18,550] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 01:29:33,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 01:29:34,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 01:29:35,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 01:29:35,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 01:29:45,965] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 01:33:54,476] INFO : Step 20100/50000 rank2 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0772 | Regression_eta: 0.0645 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0753 | LR: 6.86e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.4724s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank2 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank2 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0797 | Regression_eta: 0.0607 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0703 | LR: 6.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9220s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank2 [2026-10-29 01:41:55,631] INFO : Step 20300/50000 rank2 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0892 | Regression_eta: 0.0612 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0860 | LR: 6.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6966s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank2 [2026-10-29 01:45:46,289] INFO : Step 20400/50000 rank2 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0891 | Regression_eta: 0.0568 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0563 | LR: 6.78e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1297s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank2 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank2 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0760 | Regression_eta: 0.0549 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0691 | LR: 6.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3407s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank2 [2026-10-29 01:53:32,129] INFO : Step 20600/50000 rank2 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0675 | Regression_energy: 0.1028 | Regression_eta: 0.0573 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0738 | LR: 6.73e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.0696s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank2 [2026-10-29 01:57:29,940] INFO : Step 20700/50000 rank2 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2285 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0752 | Regression_eta: 0.0692 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0977 | LR: 6.70e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.6607s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank2 [2026-10-29 02:01:35,425] INFO : Step 20800/50000 rank2 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2469 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0891 | Regression_eta: 0.0700 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0814 | LR: 6.67e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8000s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank2 [2026-10-29 02:05:49,595] INFO : Step 20900/50000 rank2 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0854 | Regression_eta: 0.0589 | Regression_pt: 0.0876 | Regression_sin_phi: 0.1003 | LR: 6.65e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.1305s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank2 [2026-10-29 02:09:53,644] INFO : Step 21000/50000 rank2 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0946 | Regression_eta: 0.0627 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0973 | LR: 6.62e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.8423s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank2 [2026-10-29 02:09:54,611] INFO : Running validation on rank2 +rank2 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:10:45,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:10:45,899] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 02:11:12,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:11:13,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:11:14,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:11:15,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:11:25,520] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:11:26,308] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 02:11:43,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:11:44,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:11:45,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:11:46,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:11:56,654] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 02:11:57,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:11:57,983] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 02:12:13,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:12:14,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:12:15,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:12:15,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:12:25,719] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 02:16:43,091] INFO : Step 21100/50000 rank2 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0902 | Classification_binary: 0.2509 | Regression_cos_phi: 0.1191 | Regression_energy: 0.0874 | Regression_eta: 0.0986 | Regression_pt: 0.0910 | Regression_sin_phi: 0.1049 | LR: 6.59e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5459s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank2 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank2 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank2 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0892 | Regression_eta: 0.0697 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0726 | LR: 6.56e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1633s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank2 [2026-10-29 02:24:40,426] INFO : Step 21300/50000 rank2 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2494 | Regression_cos_phi: 0.1034 | Regression_energy: 0.0885 | Regression_eta: 0.0636 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0975 | LR: 6.54e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1541s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank2 [2026-10-29 02:28:35,145] INFO : Step 21400/50000 rank2 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2581 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0781 | Regression_eta: 0.0804 | Regression_pt: 0.0822 | Regression_sin_phi: 0.1000 | LR: 6.51e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4140s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank2 [2026-10-29 02:32:39,449] INFO : Step 21500/50000 rank2 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2153 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0938 | Regression_eta: 0.0967 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0820 | LR: 6.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7864s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank2 [2026-10-29 02:36:35,118] INFO : Step 21600/50000 rank2 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0756 | Regression_eta: 0.0739 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0690 | LR: 6.45e-04 | DataLoad Time: 0.0229s | Model Forward Time: 3.8424s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank2 [2026-10-29 02:40:30,747] INFO : Step 21700/50000 rank2 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2525 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0893 | Regression_eta: 0.0899 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0739 | LR: 6.43e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.5968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank2 [2026-10-29 02:44:20,672] INFO : Step 21800/50000 rank2 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2446 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0889 | Regression_eta: 0.0631 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0917 | LR: 6.40e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1244s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank2 [2026-10-29 02:48:12,872] INFO : Step 21900/50000 rank2 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0970 | Regression_eta: 0.0496 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0633 | LR: 6.37e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2320s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank2 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank2 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2140 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0870 | Regression_eta: 0.0704 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0753 | LR: 6.34e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1549s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank2 [2026-10-29 02:52:09,891] INFO : Running validation on rank2 +rank2 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:52:57,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:52:58,580] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 02:53:25,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:53:27,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:53:28,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:53:29,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:53:39,581] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 02:53:56,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:53:57,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:53:58,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:53:59,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:54:09,117] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:54:10,679] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 02:54:25,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:54:26,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:54:27,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:54:27,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:54:37,756] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank2 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 0.3201 | Regression_cos_phi: 0.1542 | Regression_energy: 0.0990 | Regression_eta: 0.0857 | Regression_pt: 0.1018 | Regression_sin_phi: 0.1245 | LR: 6.32e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4405s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank2 [2026-10-29 03:02:36,687] INFO : Step 22200/50000 rank2 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0811 | Regression_energy: 0.1045 | Regression_eta: 0.0646 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0888 | LR: 6.29e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6563s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank2 [2026-10-29 03:06:28,833] INFO : Step 22300/50000 rank2 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2132 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0960 | Regression_eta: 0.0743 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0814 | LR: 6.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8516s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank2 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank2 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2471 | Regression_cos_phi: 0.0950 | Regression_energy: 0.1163 | Regression_eta: 0.0894 | Regression_pt: 0.1231 | Regression_sin_phi: 0.1190 | LR: 6.23e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5924s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank2 [2026-10-29 03:14:16,070] INFO : Step 22500/50000 rank2 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0959 | Regression_energy: 0.1022 | Regression_eta: 0.0814 | Regression_pt: 0.1054 | Regression_sin_phi: 0.1035 | LR: 6.20e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9001s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank2 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank2 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2094 | Regression_cos_phi: 0.0974 | Regression_energy: 0.1002 | Regression_eta: 0.1052 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0923 | LR: 6.18e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5802s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank2 [2026-10-29 03:22:04,588] INFO : Step 22700/50000 rank2 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2536 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0939 | Regression_eta: 0.0794 | Regression_pt: 0.0951 | Regression_sin_phi: 0.1219 | LR: 6.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3265s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank2 [2026-10-29 03:26:06,869] INFO : Step 22800/50000 rank2 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0987 | Regression_energy: 0.1029 | Regression_eta: 0.0688 | Regression_pt: 0.1049 | Regression_sin_phi: 0.1092 | LR: 6.12e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2781s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank2 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank2 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1066 | Regression_eta: 0.0820 | Regression_pt: 0.1103 | Regression_sin_phi: 0.1001 | LR: 6.09e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.6092s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank2 [2026-10-29 03:34:03,730] INFO : Step 23000/50000 rank2 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2359 | Regression_cos_phi: 0.1276 | Regression_energy: 0.1242 | Regression_eta: 0.0975 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0941 | LR: 6.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6390s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank2 [2026-10-29 03:34:04,743] INFO : Running validation on rank2 +rank2 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 03:34:52,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:34:53,013] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 03:35:18,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 03:35:19,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 03:35:20,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 03:35:21,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 03:35:31,778] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:35:32,955] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 03:35:48,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 03:35:49,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 03:35:50,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 03:35:51,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 03:36:01,676] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 03:36:02,378] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:36:02,912] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 03:36:18,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 03:36:19,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 03:36:19,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 03:36:20,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 03:36:30,342] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 03:40:33,162] INFO : Step 23100/50000 rank2 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2722 | Regression_cos_phi: 0.1195 | Regression_energy: 0.0876 | Regression_eta: 0.1019 | Regression_pt: 0.0921 | Regression_sin_phi: 0.1293 | LR: 6.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4148s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank2 [2026-10-29 03:44:14,416] INFO : Step 23200/50000 rank2 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.1946 | Regression_cos_phi: 0.1186 | Regression_energy: 0.0938 | Regression_eta: 0.1017 | Regression_pt: 0.0968 | Regression_sin_phi: 0.1267 | LR: 6.01e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.4840s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank2 [2026-10-29 03:47:55,309] INFO : Step 23300/50000 rank2 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2105 | Regression_cos_phi: 0.1060 | Regression_energy: 0.1283 | Regression_eta: 0.0971 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0914 | LR: 5.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9637s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank2 [2026-10-29 03:51:33,797] INFO : Step 23400/50000 rank2 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2519 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0951 | Regression_eta: 0.0831 | Regression_pt: 0.0992 | Regression_sin_phi: 0.1105 | LR: 5.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1974s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank2 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank2 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0957 | Regression_eta: 0.0743 | Regression_pt: 0.0985 | Regression_sin_phi: 0.1042 | LR: 5.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0135s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank2 [2026-10-29 03:59:15,167] INFO : Step 23600/50000 rank2 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0936 | Regression_eta: 0.0643 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0674 | LR: 5.90e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9380s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank2 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank2 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2101 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0805 | Regression_eta: 0.0564 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0658 | LR: 5.87e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3846s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank2 [2026-10-29 04:06:50,322] INFO : Step 23800/50000 rank2 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2430 | Regression_cos_phi: 0.0892 | Regression_energy: 0.1129 | Regression_eta: 0.0943 | Regression_pt: 0.1170 | Regression_sin_phi: 0.1030 | LR: 5.84e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.0030s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank2 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank2 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0991 | Regression_eta: 0.0623 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0824 | LR: 5.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3376s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank2 [2026-10-29 04:14:57,720] INFO : Step 24000/50000 rank2 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0736 | Regression_energy: 0.1001 | Regression_eta: 0.0796 | Regression_pt: 0.1081 | Regression_sin_phi: 0.1387 | LR: 5.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1432s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank2 [2026-10-29 04:14:58,739] INFO : Running validation on rank2 +rank2 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:15:47,910] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:15:48,539] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 04:16:15,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:16:16,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:16:17,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:16:18,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:16:28,525] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:16:29,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 04:16:45,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:16:46,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:16:47,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:16:48,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:16:58,597] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:16:59,583] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 04:17:15,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:17:15,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:17:16,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:17:16,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:17:27,002] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 04:21:44,039] INFO : Step 24100/50000 rank2 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2724 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1051 | Regression_eta: 0.0894 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1130 | LR: 5.75e-04 | DataLoad Time: 0.0197s | Model Forward Time: 7.7537s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank2 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank2 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0919 | Regression_eta: 0.0653 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0941 | LR: 5.73e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2951s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank2 [2026-10-29 04:29:22,270] INFO : Step 24300/50000 rank2 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0969 | Regression_eta: 0.0504 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0752 | LR: 5.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0414s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank2 [2026-10-29 04:33:07,494] INFO : Step 24400/50000 rank2 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2554 | Regression_cos_phi: 0.1044 | Regression_energy: 0.1206 | Regression_eta: 0.0779 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0954 | LR: 5.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2374s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank2 [2026-10-29 04:36:57,191] INFO : Step 24500/50000 rank2 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.3106 | Regression_cos_phi: 0.1207 | Regression_energy: 0.1234 | Regression_eta: 0.1100 | Regression_pt: 0.1291 | Regression_sin_phi: 0.1359 | LR: 5.64e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3264s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank2 [2026-10-29 04:40:40,561] INFO : Step 24600/50000 rank2 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1997 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1090 | Regression_eta: 0.0822 | Regression_pt: 0.1132 | Regression_sin_phi: 0.1095 | LR: 5.61e-04 | DataLoad Time: 0.0260s | Model Forward Time: 3.9889s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank2 [2026-10-29 04:44:29,988] INFO : Step 24700/50000 rank2 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0915 | Classification_binary: 0.2609 | Regression_cos_phi: 0.1090 | Regression_energy: 0.1242 | Regression_eta: 0.1246 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1232 | LR: 5.58e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5081s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank2 [2026-10-29 04:48:15,358] INFO : Step 24800/50000 rank2 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2875 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0992 | Regression_eta: 0.0829 | Regression_pt: 0.1035 | Regression_sin_phi: 0.1359 | LR: 5.56e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8657s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank2 [2026-10-29 04:52:00,597] INFO : Step 24900/50000 rank2 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0690 | Regression_energy: 0.1017 | Regression_eta: 0.0679 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0873 | LR: 5.53e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.9771s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank2 [2026-10-29 04:55:44,333] INFO : Step 25000/50000 rank2 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2293 | Regression_cos_phi: 0.1008 | Regression_energy: 0.1131 | Regression_eta: 0.0779 | Regression_pt: 0.1192 | Regression_sin_phi: 0.1261 | LR: 5.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2483s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank2 [2026-10-29 04:55:45,304] INFO : Running validation on rank2 +rank2 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:56:33,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:56:34,567] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 04:57:00,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:57:01,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:57:02,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:57:03,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:57:13,293] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:57:13,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:57:13,950] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 04:57:29,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:57:30,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:57:31,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:57:32,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:57:42,975] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:57:44,417] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 04:57:59,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:58:00,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:58:01,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:58:01,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:58:11,948] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 05:02:15,911] INFO : Step 25100/50000 rank2 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2849 | Regression_cos_phi: 0.0907 | Regression_energy: 0.1058 | Regression_eta: 0.1009 | Regression_pt: 0.1113 | Regression_sin_phi: 0.1185 | LR: 5.47e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.7055s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank2 [2026-10-29 05:06:02,045] INFO : Step 25200/50000 rank2 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0952 | Regression_eta: 0.0745 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0863 | LR: 5.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2815s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank2 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank2 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0558 | Regression_energy: 0.1102 | Regression_eta: 0.0495 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0514 | LR: 5.42e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8571s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank2 [2026-10-29 05:13:35,656] INFO : Step 25400/50000 rank2 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2597 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1094 | Regression_eta: 0.0852 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1131 | LR: 5.39e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0667s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank2 [2026-10-29 05:17:27,535] INFO : Step 25500/50000 rank2 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1043 | Regression_energy: 0.1020 | Regression_eta: 0.0745 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0874 | LR: 5.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0767s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank2 [2026-10-29 05:21:16,672] INFO : Step 25600/50000 rank2 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0728 | Regression_energy: 0.1221 | Regression_eta: 0.0756 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1035 | LR: 5.33e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8763s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank2 [2026-10-29 05:25:00,204] INFO : Step 25700/50000 rank2 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2368 | Regression_cos_phi: 0.1088 | Regression_energy: 0.1128 | Regression_eta: 0.0782 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0866 | LR: 5.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8873s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank2 [2026-10-29 05:28:44,679] INFO : Step 25800/50000 rank2 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.3292 | Regression_cos_phi: 0.1293 | Regression_energy: 0.1368 | Regression_eta: 0.1421 | Regression_pt: 0.1451 | Regression_sin_phi: 0.1480 | LR: 5.27e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5184s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank2 [2026-10-29 05:32:33,909] INFO : Step 25900/50000 rank2 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2317 | Regression_cos_phi: 0.0875 | Regression_energy: 0.1070 | Regression_eta: 0.0817 | Regression_pt: 0.1130 | Regression_sin_phi: 0.1240 | LR: 5.25e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8739s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank2 [2026-10-29 05:36:28,636] INFO : Step 26000/50000 rank2 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.1933 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1100 | Regression_eta: 0.1043 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0704 | LR: 5.22e-04 | DataLoad Time: 0.0186s | Model Forward Time: 9.2325s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank2 [2026-10-29 05:36:29,552] INFO : Running validation on rank2 +rank2 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 05:37:19,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:37:20,019] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 05:37:45,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 05:37:46,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 05:37:47,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 05:37:48,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 05:37:58,026] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 05:37:58,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:37:59,110] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 05:38:15,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 05:38:16,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 05:38:17,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 05:38:18,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 05:38:28,040] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:38:28,892] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 05:38:44,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 05:38:45,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 05:38:45,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 05:38:46,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 05:38:56,408] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 05:43:04,442] INFO : Step 26100/50000 rank2 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2540 | Regression_cos_phi: 0.1554 | Regression_energy: 0.1239 | Regression_eta: 0.0947 | Regression_pt: 0.1288 | Regression_sin_phi: 0.1700 | LR: 5.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8571s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank2 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank2 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0842 | Regression_energy: 0.1030 | Regression_eta: 0.0721 | Regression_pt: 0.1058 | Regression_sin_phi: 0.1319 | LR: 5.16e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.2696s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank2 [2026-10-29 05:50:52,485] INFO : Step 26300/50000 rank2 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0863 | Regression_energy: 0.1162 | Regression_eta: 0.0776 | Regression_pt: 0.1221 | Regression_sin_phi: 0.1094 | LR: 5.13e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.3841s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank2 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank2 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0933 | Classification_binary: 0.2307 | Regression_cos_phi: 0.0795 | Regression_energy: 0.1174 | Regression_eta: 0.0824 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1161 | LR: 5.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3820s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank2 [2026-10-29 05:58:42,121] INFO : Step 26500/50000 rank2 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2459 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1081 | Regression_eta: 0.0887 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0917 | LR: 5.08e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4062s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank2 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank2 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0766 | Regression_energy: 0.1125 | Regression_eta: 0.0900 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1107 | LR: 5.05e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.7625s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank2 [2026-10-29 06:06:39,600] INFO : Step 26700/50000 rank2 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0974 | Classification_binary: 0.2489 | Regression_cos_phi: 0.1105 | Regression_energy: 0.1212 | Regression_eta: 0.1215 | Regression_pt: 0.1265 | Regression_sin_phi: 0.1228 | LR: 5.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.6709s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank2 [2026-10-29 06:10:41,254] INFO : Step 26800/50000 rank2 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2665 | Regression_cos_phi: 0.1236 | Regression_energy: 0.1155 | Regression_eta: 0.0900 | Regression_pt: 0.1201 | Regression_sin_phi: 0.1295 | LR: 4.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2395s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank2 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank2 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2208 | Regression_cos_phi: 0.0662 | Regression_energy: 0.1132 | Regression_eta: 0.0735 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0630 | LR: 4.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8919s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank2 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank2 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2184 | Regression_cos_phi: 0.0668 | Regression_energy: 0.1070 | Regression_eta: 0.0629 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0968 | LR: 4.94e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9881s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank2 [2026-10-29 06:18:43,933] INFO : Running validation on rank2 +rank2 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 06:19:32,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:19:33,149] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 06:20:00,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 06:20:01,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 06:20:02,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 06:20:03,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 06:20:13,628] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:20:14,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:20:14,780] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 06:20:30,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 06:20:32,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 06:20:33,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 06:20:34,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 06:20:44,108] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:20:46,293] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 06:21:01,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 06:21:02,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 06:21:03,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 06:21:03,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 06:21:13,790] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 06:25:28,750] INFO : Step 27100/50000 rank2 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0955 | Regression_energy: 0.1192 | Regression_eta: 0.0688 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1333 | LR: 4.91e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6995s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank2 [2026-10-29 06:29:29,296] INFO : Step 27200/50000 rank2 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2588 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1157 | Regression_eta: 0.0749 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0906 | LR: 4.88e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5768s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank2 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank2 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0962 | Regression_eta: 0.0727 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0746 | LR: 4.85e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.9946s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank2 [2026-10-29 06:37:36,742] INFO : Step 27400/50000 rank2 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2090 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0980 | Regression_eta: 0.0521 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0800 | LR: 4.82e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5101s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank2 [2026-10-29 06:41:28,998] INFO : Step 27500/50000 rank2 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2983 | Regression_cos_phi: 0.0923 | Regression_energy: 0.1189 | Regression_eta: 0.0945 | Regression_pt: 0.1256 | Regression_sin_phi: 0.1047 | LR: 4.80e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9222s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank2 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank2 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2379 | Regression_cos_phi: 0.1383 | Regression_energy: 0.1001 | Regression_eta: 0.0750 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0965 | LR: 4.77e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8539s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank2 [2026-10-29 06:49:16,015] INFO : Step 27700/50000 rank2 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0814 | Regression_energy: 0.1410 | Regression_eta: 0.0667 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0725 | LR: 4.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0204s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank2 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank2 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0828 | Regression_energy: 0.1264 | Regression_eta: 0.0760 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0759 | LR: 4.71e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.6684s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank2 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank2 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2308 | Regression_cos_phi: 0.0816 | Regression_energy: 0.1002 | Regression_eta: 0.0784 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0894 | LR: 4.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5360s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank2 [2026-10-29 07:01:10,308] INFO : Step 28000/50000 rank2 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1945 | Regression_cos_phi: 0.1007 | Regression_energy: 0.1006 | Regression_eta: 0.0817 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0998 | LR: 4.66e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3457s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank2 [2026-10-29 07:01:11,148] INFO : Running validation on rank2 +rank2 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:02:01,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:02:02,349] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 07:02:30,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:02:30,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:02:31,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:02:32,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:02:42,946] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:02:44,046] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 07:02:59,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:03:00,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:03:01,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:03:02,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:03:12,524] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 07:03:13,952] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:03:14,472] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 07:03:30,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:03:30,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:03:31,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:03:32,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:03:42,121] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 07:08:04,579] INFO : Step 28100/50000 rank2 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2555 | Regression_cos_phi: 0.0828 | Regression_energy: 0.1182 | Regression_eta: 0.0707 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0928 | LR: 4.63e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank2 [2026-10-29 07:11:56,590] INFO : Step 28200/50000 rank2 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2438 | Regression_cos_phi: 0.0696 | Regression_energy: 0.1066 | Regression_eta: 0.0592 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0906 | LR: 4.60e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank2 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank2 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0907 | Regression_energy: 0.1222 | Regression_eta: 0.0836 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0840 | LR: 4.57e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8325s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank2 [2026-10-29 07:20:03,895] INFO : Step 28400/50000 rank2 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2028 | Regression_cos_phi: 0.0969 | Regression_energy: 0.1123 | Regression_eta: 0.0740 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0971 | LR: 4.55e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.0895s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank2 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank2 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2467 | Regression_cos_phi: 0.0879 | Regression_energy: 0.1243 | Regression_eta: 0.0675 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0895 | LR: 4.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1337s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank2 [2026-10-29 07:27:52,535] INFO : Step 28600/50000 rank2 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2312 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0935 | Regression_eta: 0.0921 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0902 | LR: 4.49e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2897s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank2 [2026-10-29 07:31:49,173] INFO : Step 28700/50000 rank2 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0942 | Regression_eta: 0.0608 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0879 | LR: 4.46e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0937s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank2 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank2 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0944 | Regression_eta: 0.0640 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0980 | LR: 4.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank2 [2026-10-29 07:39:38,710] INFO : Step 28900/50000 rank2 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0976 | Regression_eta: 0.0604 | Regression_pt: 0.0999 | Regression_sin_phi: 0.1099 | LR: 4.41e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3189s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank2 [2026-10-29 07:43:43,866] INFO : Step 29000/50000 rank2 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0613 | Regression_energy: 0.1041 | Regression_eta: 0.0582 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0599 | LR: 4.38e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9071s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank2 [2026-10-29 07:43:44,819] INFO : Running validation on rank2 +rank2 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:44:33,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:44:33,713] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 07:44:59,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:45:00,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:45:01,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:45:02,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:45:12,398] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:45:12,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:45:13,372] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 07:45:29,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:45:30,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:45:31,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:45:32,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:45:42,565] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:45:43,719] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 07:45:59,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:46:00,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:46:00,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:46:01,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:46:11,331] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 07:50:42,138] INFO : Step 29100/50000 rank2 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2360 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0994 | Regression_eta: 0.0641 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0904 | LR: 4.35e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6982s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank2 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank2 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2523 | Regression_cos_phi: 0.1140 | Regression_energy: 0.0909 | Regression_eta: 0.1101 | Regression_pt: 0.0955 | Regression_sin_phi: 0.1349 | LR: 4.33e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.2681s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank2 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank2 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0907 | Regression_eta: 0.0475 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0623 | LR: 4.30e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9996s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank2 [2026-10-29 08:02:11,407] INFO : Step 29400/50000 rank2 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2174 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0921 | Regression_eta: 0.0806 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0875 | LR: 4.27e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5179s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank2 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank2 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2266 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0915 | Regression_eta: 0.0571 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0883 | LR: 4.24e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4986s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank2 [2026-10-29 08:09:49,626] INFO : Step 29600/50000 rank2 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0844 | Regression_eta: 0.0867 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0997 | LR: 4.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7092s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank2 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank2 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0870 | Regression_eta: 0.0629 | Regression_pt: 0.0908 | Regression_sin_phi: 0.1111 | LR: 4.19e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1833s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank2 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank2 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2312 | Regression_cos_phi: 0.0915 | Regression_energy: 0.1000 | Regression_eta: 0.0663 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0948 | LR: 4.16e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3854s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank2 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank2 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.2342 | Regression_cos_phi: 0.1066 | Regression_energy: 0.1033 | Regression_eta: 0.0744 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0722 | LR: 4.14e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.9967s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank2 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank2 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0978 | Regression_eta: 0.0704 | Regression_pt: 0.1011 | Regression_sin_phi: 0.1268 | LR: 4.11e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7038s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank2 [2026-10-29 08:24:57,340] INFO : Running validation on rank2 +rank2 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 08:25:45,090] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:25:45,652] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 08:26:11,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 08:26:12,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 08:26:13,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 08:26:14,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 08:26:24,305] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 08:26:24,835] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:26:25,382] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 08:26:41,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 08:26:42,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 08:26:43,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 08:26:44,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 08:26:54,458] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:26:55,053] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 08:27:10,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 08:27:11,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 08:27:12,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 08:27:12,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 08:27:22,944] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank2 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0806 | Regression_eta: 0.0604 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0913 | LR: 4.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8517s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank2 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank2 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1011 | Regression_eta: 0.0736 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1185 | LR: 4.06e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.5318s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank2 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank2 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0803 | Regression_energy: 0.1024 | Regression_eta: 0.0638 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0587 | LR: 4.03e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.1780s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank2 [2026-10-29 08:43:19,167] INFO : Step 30400/50000 rank2 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0990 | Regression_eta: 0.0591 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0677 | LR: 4.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5068s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank2 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank2 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2369 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0834 | Regression_eta: 0.0666 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1085 | LR: 3.98e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7627s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank2 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank2 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0959 | Regression_energy: 0.1025 | Regression_eta: 0.0584 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0834 | LR: 3.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5470s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank2 [2026-10-29 08:54:35,602] INFO : Step 30700/50000 rank2 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0876 | Regression_eta: 0.0723 | Regression_pt: 0.0937 | Regression_sin_phi: 0.1099 | LR: 3.92e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2953s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank2 [2026-10-29 08:58:20,946] INFO : Step 30800/50000 rank2 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0878 | Regression_eta: 0.0739 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1053 | LR: 3.90e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2077s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank2 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank2 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0838 | Regression_eta: 0.0568 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0585 | LR: 3.87e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9301s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank2 [2026-10-29 09:05:52,366] INFO : Step 31000/50000 rank2 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0831 | Regression_eta: 0.0567 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0824 | LR: 3.84e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.1888s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank2 [2026-10-29 09:05:53,352] INFO : Running validation on rank2 +rank2 [2026-10-29 09:05:53,352] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:05:53,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:06:41,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:06:42,227] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 09:07:08,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:07:09,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:07:10,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:07:11,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:07:21,485] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:07:22,582] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 09:07:38,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:07:39,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:07:40,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:07:41,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:07:51,018] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:07:52,509] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 09:08:07,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:08:08,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:08:09,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:08:09,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:08:19,886] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank2 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0762 | Regression_eta: 0.0567 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0656 | LR: 3.82e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4605s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank2 [2026-10-29 09:16:29,207] INFO : Step 31200/50000 rank2 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2310 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0899 | Regression_eta: 0.0584 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0865 | LR: 3.79e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5362s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank2 [2026-10-29 09:20:24,830] INFO : Step 31300/50000 rank2 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2207 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0874 | Regression_eta: 0.1045 | Regression_pt: 0.0885 | Regression_sin_phi: 0.1155 | LR: 3.76e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.4678s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank2 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank2 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0793 | Regression_eta: 0.0529 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0653 | LR: 3.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8368s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank2 [2026-10-29 09:28:17,180] INFO : Step 31500/50000 rank2 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2157 | Regression_cos_phi: 0.1024 | Regression_energy: 0.0944 | Regression_eta: 0.0945 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0814 | LR: 3.71e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3622s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank2 [2026-10-29 09:32:16,797] INFO : Step 31600/50000 rank2 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0720 | Regression_eta: 0.0779 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0857 | LR: 3.69e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8629s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank2 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank2 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0994 | Regression_energy: 0.0778 | Regression_eta: 0.0711 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0757 | LR: 3.66e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1365s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank2 [2026-10-29 09:40:22,622] INFO : Step 31800/50000 rank2 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0867 | Regression_eta: 0.0522 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0721 | LR: 3.64e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3532s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank2 [2026-10-29 09:48:15,345] INFO : Step 31900/50000 rank2 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0648 | Regression_eta: 0.0407 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0725 | LR: 3.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8698s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank2 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank2 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0928 | Regression_eta: 0.0533 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0680 | LR: 3.58e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1239s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank2 [2026-10-29 09:52:26,792] INFO : Running validation on rank2 +rank2 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:53:17,424] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 09:53:46,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:53:47,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:53:49,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:53:50,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:54:00,034] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:54:01,008] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 09:54:17,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:54:18,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:54:19,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:54:20,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:54:30,019] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:54:30,788] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 09:54:46,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:54:47,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:54:47,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:54:48,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:54:58,435] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank2 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0900 | Regression_eta: 0.0623 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0863 | LR: 3.56e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.9461s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank2 [2026-10-29 10:03:26,459] INFO : Step 32200/50000 rank2 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0913 | Regression_eta: 0.0740 | Regression_pt: 0.0952 | Regression_sin_phi: 0.1216 | LR: 3.53e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5226s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank2 [2026-10-29 10:07:22,792] INFO : Step 32300/50000 rank2 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0830 | Regression_eta: 0.0513 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0648 | LR: 3.51e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5896s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank2 [2026-10-29 10:11:42,986] INFO : Step 32400/50000 rank2 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0828 | Regression_eta: 0.0516 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0604 | LR: 3.48e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.0255s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank2 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank2 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0629 | Regression_energy: 0.1084 | Regression_eta: 0.0494 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0823 | LR: 3.46e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.5256s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank2 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank2 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2193 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0812 | Regression_eta: 0.0519 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0865 | LR: 3.43e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4765s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank2 [2026-10-29 10:24:11,749] INFO : Step 32700/50000 rank2 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0836 | Regression_eta: 0.0493 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0638 | LR: 3.41e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0503s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank2 [2026-10-29 10:28:05,809] INFO : Step 32800/50000 rank2 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0804 | Regression_eta: 0.0602 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0583 | LR: 3.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5508s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank2 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank2 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2255 | Regression_cos_phi: 0.1469 | Regression_energy: 0.0833 | Regression_eta: 0.0752 | Regression_pt: 0.0866 | Regression_sin_phi: 0.1042 | LR: 3.36e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8075s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank2 [2026-10-29 10:36:14,933] INFO : Step 33000/50000 rank2 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0826 | Regression_eta: 0.0729 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0938 | LR: 3.33e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7654s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank2 [2026-10-29 10:36:15,962] INFO : Running validation on rank2 +rank2 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 10:37:03,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:37:04,371] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 10:37:32,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 10:37:34,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 10:37:36,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 10:37:37,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 10:37:47,550] INFO : Time taken to make predictions on device 2 is: 0.30 min +rank2 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:37:48,813] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 10:38:07,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 10:38:09,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 10:38:10,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 10:38:11,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 10:38:21,017] INFO : Time taken to make predictions on device 2 is: 0.30 min +rank2 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:38:21,708] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 10:38:37,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 10:38:38,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 10:38:38,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 10:38:39,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 10:38:49,292] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 10:43:12,058] INFO : Step 33100/50000 rank2 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0750 | Regression_eta: 0.0655 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0602 | LR: 3.31e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.9500s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank2 [2026-10-29 10:47:22,843] INFO : Step 33200/50000 rank2 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0764 | Regression_eta: 0.0692 | Regression_pt: 0.0793 | Regression_sin_phi: 0.1234 | LR: 3.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6971s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank2 [2026-10-29 10:51:33,085] INFO : Step 33300/50000 rank2 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0821 | Regression_eta: 0.0505 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0776 | LR: 3.26e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4502s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank2 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank2 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0677 | Regression_eta: 0.0606 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0676 | LR: 3.23e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5445s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank2 [2026-10-29 10:59:48,622] INFO : Step 33500/50000 rank2 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0756 | Regression_eta: 0.0698 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0870 | LR: 3.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.9159s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank2 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank2 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2401 | Regression_cos_phi: 0.1026 | Regression_energy: 0.0937 | Regression_eta: 0.0828 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0968 | LR: 3.19e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2014s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank2 [2026-10-29 11:09:08,678] INFO : Step 33700/50000 rank2 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0820 | Regression_eta: 0.0560 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0649 | LR: 3.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3158s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank2 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank2 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0867 | Regression_eta: 0.0837 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0909 | LR: 3.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3669s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank2 [2026-10-29 11:16:52,540] INFO : Step 33900/50000 rank2 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0741 | Regression_eta: 0.0658 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0671 | LR: 3.11e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.8118s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank2 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank2 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0775 | Regression_eta: 0.0872 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0569 | LR: 3.09e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.9491s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank2 [2026-10-29 11:20:48,852] INFO : Running validation on rank2 +rank2 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 11:21:42,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:21:42,910] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 11:22:09,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 11:22:10,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 11:22:11,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 11:22:12,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 11:22:22,249] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:22:22,977] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 11:22:39,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 11:22:40,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 11:22:41,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 11:22:42,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 11:22:52,025] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:22:53,477] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 11:23:09,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 11:23:09,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 11:23:10,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 11:23:11,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 11:23:21,226] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 11:27:42,516] INFO : Step 34100/50000 rank2 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0847 | Regression_eta: 0.0706 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0947 | LR: 3.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 7.5175s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank2 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank2 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1959 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0776 | Regression_eta: 0.0360 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0591 | LR: 3.04e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7870s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank2 [2026-10-29 11:35:56,838] INFO : Step 34300/50000 rank2 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0763 | Regression_eta: 0.0575 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0772 | LR: 3.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.2605s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank2 [2026-10-29 11:40:08,369] INFO : Step 34400/50000 rank2 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0959 | Regression_eta: 0.0412 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0472 | LR: 2.99e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.9075s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank2 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank2 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0870 | Regression_eta: 0.0609 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0666 | LR: 2.97e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5061s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank2 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank2 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2438 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0752 | Regression_eta: 0.0864 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0742 | LR: 2.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6055s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank2 [2026-10-29 11:52:00,074] INFO : Step 34700/50000 rank2 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0798 | Regression_eta: 0.0459 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0492 | LR: 2.92e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.5506s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank2 [2026-10-29 11:55:53,802] INFO : Step 34800/50000 rank2 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0805 | Regression_eta: 0.0330 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0578 | LR: 2.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8586s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank2 [2026-10-29 11:59:58,709] INFO : Step 34900/50000 rank2 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2150 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0796 | Regression_eta: 0.0572 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0849 | LR: 2.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1706s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank2 [2026-10-29 12:04:06,318] INFO : Step 35000/50000 rank2 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2099 | Regression_cos_phi: 0.1049 | Regression_energy: 0.0811 | Regression_eta: 0.0692 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0961 | LR: 2.85e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4589s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank2 [2026-10-29 12:04:07,348] INFO : Running validation on rank2 +rank2 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 12:05:04,842] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:05:05,388] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 12:05:31,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 12:05:32,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 12:05:33,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 12:05:34,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 12:05:44,842] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 12:06:02,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 12:06:03,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 12:06:03,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 12:06:04,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 12:06:14,860] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 12:06:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:06:16,077] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 12:06:31,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 12:06:32,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 12:06:33,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 12:06:33,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 12:06:43,794] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank2 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0871 | Regression_eta: 0.0571 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0678 | LR: 2.83e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.6529s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank2 [2026-10-29 12:15:06,104] INFO : Step 35200/50000 rank2 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0783 | Regression_eta: 0.0528 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0642 | LR: 2.81e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.0459s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank2 [2026-10-29 12:18:57,446] INFO : Step 35300/50000 rank2 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0982 | Regression_eta: 0.0459 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0662 | LR: 2.79e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.8170s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank2 [2026-10-29 12:23:00,202] INFO : Step 35400/50000 rank2 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0775 | Regression_eta: 0.0472 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0471 | LR: 2.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3954s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank2 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank2 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0987 | Regression_energy: 0.0929 | Regression_eta: 0.0638 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0615 | LR: 2.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2681s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank2 [2026-10-29 12:31:17,345] INFO : Step 35600/50000 rank2 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2306 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0941 | Regression_eta: 0.0822 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0928 | LR: 2.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1673s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank2 [2026-10-29 12:35:31,257] INFO : Step 35700/50000 rank2 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0828 | Regression_eta: 0.0555 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0483 | LR: 2.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2783s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank2 [2026-10-29 14:49:26,323] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-29 14:49:26,335] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank2 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-29 14:49:53,253] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank2 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank2 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank2 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank2 [2026-10-29 14:49:53,279] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,282] INFO : Initializing backbone layer 3 +rank2 [2026-10-29 14:49:53,282] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,285] INFO : Initializing backbone layer 4 +rank2 [2026-10-29 14:49:53,285] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,288] INFO : Initializing backbone layer 5 +rank2 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank2 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank2 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank2 [2026-10-29 14:49:53,341] INFO : Output DNNs initialization took 0.05s +rank2 [2026-10-29 14:49:53,341] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-29 14:49:53,341] INFO : jagged_attention enabled=True +rank2 [2026-10-29 14:49:53,341] INFO : flash_attn_varlen enabled=True +rank2 [2026-10-29 14:49:53,341] INFO : task_queries enabled=False +rank2 [2026-10-29 14:49:53,341] INFO : classification_norm parameters: 0 +rank2 [2026-10-29 14:49:53,341] INFO : regression_norm parameters: 0 +rank2 [2026-10-29 14:49:53,341] INFO : classification_query parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank2 [2026-10-29 14:49:53,342] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-29 14:49:53,342] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_pid parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_pt parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_eta parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-29 14:49:53,342] INFO : nn_energy parameters: 0 +rank2 [2026-10-29 14:49:53,343] INFO : Total MLPF parameters: 7260244 +rank2 [2026-10-29 14:49:53,343] INFO : MLPF __init__ done +rank2 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank2 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=2 +rank2 [2026-10-29 14:49:53,361] INFO : Moved model to device rank=2 +rank2 [2026-10-29 14:49:53,362] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-29 14:49:53,684] INFO : Restoring training from step 35001 +rank2 [2026-10-29 14:49:53,690] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank2 [2026-10-29 14:49:53,690] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-29 14:49:53,909] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-29 14:49:54,026] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank2 [2026-10-29 14:49:54,026] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-29 14:49:54,027] INFO : Trainable parameters: 7260244 +rank2 [2026-10-29 14:49:54,027] INFO : Non-trainable parameters: 0 +rank2 [2026-10-29 14:49:54,027] INFO : Total parameters: 7260244 +rank2 [2026-10-29 14:49:54,030] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-29 14:49:54,030] INFO : Getting dataloaders +rank2 [2026-10-29 14:49:54,030] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:01,421] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:02,343] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:03,543] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-29 14:50:03,544] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-29 14:50:03,544] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:03,809] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:04,078] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:04,389] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-29 14:50:04,389] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-29 14:50:18,358] INFO : Got dataloaders +rank2 [2026-10-29 14:50:18,359] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank2 [2026-10-29 14:50:18,359] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank2 [2026-10-29 14:50:18,359] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank2 [2026-10-29 14:50:18,359] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank2 [2026-10-29 14:50:19,683] INFO : loader split=train rank=2 len=21108 +rank2 [2026-10-29 14:50:19,684] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-29 14:50:19,684] INFO : Starting training from step 35001 to 50000 on rank 2 of 8 +rank2 [2026-10-29 14:50:19,684] INFO : Creating train_iterator +rank2 [2026-10-29 14:50:19,684] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank2 [2026-10-29 14:50:33,498] INFO : Created train_iterator +rank2 [2026-10-29 14:54:16,364] INFO : Step 35100/50000 rank2 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1634 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0824 | Regression_eta: 0.0583 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0613 | LR: 2.83e-04 | DataLoad Time: 0.0168s | Model Forward Time: 2.8682s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank2 [2026-10-29 14:57:53,684] INFO : Step 35200/50000 rank2 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0839 | Regression_eta: 0.0414 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0658 | LR: 2.81e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5090s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank2 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank2 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0553 | Regression_energy: 0.1013 | Regression_eta: 0.0474 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0569 | LR: 2.79e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5922s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank2 [2026-10-29 15:05:10,669] INFO : Step 35400/50000 rank2 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0820 | Regression_eta: 0.0536 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0639 | LR: 2.76e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.4265s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank2 [2026-10-29 15:08:49,315] INFO : Step 35500/50000 rank2 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1894 | Regression_cos_phi: 0.1099 | Regression_energy: 0.0914 | Regression_eta: 0.0766 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0740 | LR: 2.74e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.4540s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank2 [2026-10-29 15:12:27,418] INFO : Step 35600/50000 rank2 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2293 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0976 | Regression_eta: 0.0638 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0835 | LR: 2.72e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6873s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank2 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank2 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0779 | Regression_eta: 0.0619 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0454 | LR: 2.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.3523s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank2 [2026-10-29 15:19:47,616] INFO : Step 35800/50000 rank2 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0731 | Regression_eta: 0.0651 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0553 | LR: 2.68e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.4876s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank2 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank2 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0839 | Regression_eta: 0.0520 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0704 | LR: 2.65e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8620s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank2 [2026-10-29 15:27:19,181] INFO : Step 36000/50000 rank2 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0665 | Regression_eta: 0.0664 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0609 | LR: 2.63e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8285s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank2 [2026-10-29 15:27:20,114] INFO : Running validation on rank2 +rank2 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 15:28:08,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:28:08,876] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 15:28:36,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 15:28:37,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 15:28:38,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 15:28:39,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 15:28:49,496] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 15:28:49,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:28:50,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 15:29:06,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 15:29:07,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 15:29:08,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 15:29:08,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 15:29:18,925] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:29:20,197] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 15:29:35,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 15:29:36,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 15:29:37,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 15:29:37,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 15:29:47,721] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank2 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0968 | Regression_eta: 0.0507 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0635 | LR: 2.61e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9091s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank2 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank2 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0875 | Regression_eta: 0.0640 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0983 | LR: 2.59e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5678s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank2 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank2 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0731 | Regression_eta: 0.0550 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0718 | LR: 2.57e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5114s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank2 [2026-10-29 15:45:12,807] INFO : Step 36400/50000 rank2 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0864 | Regression_eta: 0.0589 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0610 | LR: 2.55e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.2457s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank2 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank2 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0675 | Regression_eta: 0.0668 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0793 | LR: 2.52e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.0153s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank2 [2026-10-29 15:52:51,924] INFO : Step 36600/50000 rank2 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0796 | Regression_eta: 0.0653 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0720 | LR: 2.50e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7052s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank2 [2026-10-29 15:56:51,302] INFO : Step 36700/50000 rank2 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0781 | Regression_eta: 0.0848 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0876 | LR: 2.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7868s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank2 [2026-10-29 16:00:43,525] INFO : Step 36800/50000 rank2 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2027 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0906 | Regression_eta: 0.0599 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0934 | LR: 2.46e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4177s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank2 [2026-10-29 16:04:39,810] INFO : Step 36900/50000 rank2 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0888 | Regression_eta: 0.0513 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0662 | LR: 2.44e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7930s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank2 [2026-10-29 16:08:30,144] INFO : Step 37000/50000 rank2 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0792 | Regression_eta: 0.0587 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0912 | LR: 2.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9057s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank2 [2026-10-29 16:08:31,022] INFO : Running validation on rank2 +rank2 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:09:20,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:09:20,759] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 16:09:46,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:09:47,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:09:48,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:09:49,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:09:59,318] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 16:10:16,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:10:17,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:10:18,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:10:19,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:10:29,642] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:10:30,654] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 16:10:45,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:10:46,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:10:47,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:10:47,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:10:57,900] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 16:15:06,403] INFO : Step 37100/50000 rank2 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0859 | Regression_eta: 0.0447 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0623 | LR: 2.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8502s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank2 [2026-10-29 16:18:46,283] INFO : Step 37200/50000 rank2 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0804 | Regression_eta: 0.0561 | Regression_pt: 0.0837 | Regression_sin_phi: 0.1145 | LR: 2.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2299s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank2 [2026-10-29 16:22:16,729] INFO : Step 37300/50000 rank2 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0944 | Regression_eta: 0.0597 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0858 | LR: 2.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5810s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank2 [2026-10-29 16:26:01,824] INFO : Step 37400/50000 rank2 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0858 | Regression_eta: 0.0416 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0593 | LR: 2.34e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9415s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank2 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank2 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0750 | Regression_eta: 0.0706 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0930 | LR: 2.32e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3488s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank2 [2026-10-29 16:34:14,940] INFO : Step 37600/50000 rank2 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2379 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0864 | Regression_eta: 0.0728 | Regression_pt: 0.0900 | Regression_sin_phi: 0.1051 | LR: 2.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1117s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank2 [2026-10-29 16:38:00,807] INFO : Step 37700/50000 rank2 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1768 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0853 | Regression_eta: 0.0739 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0670 | LR: 2.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1665s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank2 [2026-10-29 16:41:51,410] INFO : Step 37800/50000 rank2 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0870 | Regression_eta: 0.0511 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0823 | LR: 2.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.2928s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank2 [2026-10-29 16:45:39,812] INFO : Step 37900/50000 rank2 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2266 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0749 | Regression_eta: 0.0414 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0603 | LR: 2.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8088s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank2 [2026-10-29 16:49:32,192] INFO : Step 38000/50000 rank2 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0735 | Regression_eta: 0.0999 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0838 | LR: 2.22e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8901s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank2 [2026-10-29 16:49:33,136] INFO : Running validation on rank2 +rank2 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:49:33,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:50:21,279] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 16:50:48,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:50:49,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:50:52,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:50:54,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:51:04,532] INFO : Time taken to make predictions on device 2 is: 0.31 min +rank2 [2026-10-29 16:51:06,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:51:06,743] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 16:51:22,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:51:23,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:51:24,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:51:25,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:51:35,450] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:51:36,914] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 16:51:52,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:51:53,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:51:53,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:51:54,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:52:04,372] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank2 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0828 | Regression_eta: 0.0499 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0772 | LR: 2.20e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.3705s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank2 [2026-10-29 17:00:01,013] INFO : Step 38200/50000 rank2 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2077 | Regression_cos_phi: 0.1035 | Regression_energy: 0.0871 | Regression_eta: 0.0660 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0882 | LR: 2.18e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8293s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank2 [2026-10-29 17:03:44,996] INFO : Step 38300/50000 rank2 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0900 | Regression_eta: 0.0599 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0732 | LR: 2.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2337s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank2 [2026-10-29 17:07:28,968] INFO : Step 38400/50000 rank2 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0876 | Regression_eta: 0.0347 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0603 | LR: 2.14e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5997s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank2 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank2 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0814 | Regression_eta: 0.0574 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0846 | LR: 2.12e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.5374s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank2 [2026-10-29 17:15:06,551] INFO : Step 38600/50000 rank2 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0793 | Regression_eta: 0.0425 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0731 | LR: 2.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4113s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank2 [2026-10-29 17:18:55,030] INFO : Step 38700/50000 rank2 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0873 | Regression_eta: 0.0538 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0720 | LR: 2.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.8737s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank2 [2026-10-29 17:22:44,405] INFO : Step 38800/50000 rank2 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0681 | Regression_eta: 0.0763 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0532 | LR: 2.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5787s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank2 [2026-10-29 17:26:35,865] INFO : Step 38900/50000 rank2 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0753 | Regression_eta: 0.0523 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0599 | LR: 2.05e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6663s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank2 [2026-10-29 17:30:46,441] INFO : Step 39000/50000 rank2 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0778 | Regression_eta: 0.0319 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0606 | LR: 2.03e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.6482s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank2 [2026-10-29 17:30:47,444] INFO : Running validation on rank2 +rank2 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 17:31:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:31:36,204] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 17:32:01,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 17:32:02,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 17:32:03,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 17:32:04,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 17:32:14,963] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:32:15,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 17:32:32,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 17:32:32,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 17:32:33,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 17:32:34,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 17:32:44,800] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:32:45,863] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 17:33:01,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 17:33:01,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 17:33:02,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 17:33:03,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 17:33:13,071] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank2 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0753 | Regression_eta: 0.0511 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0752 | LR: 2.01e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.6377s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank2 [2026-10-29 17:41:18,056] INFO : Step 39200/50000 rank2 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0777 | Regression_eta: 0.0399 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0577 | LR: 2.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7455s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank2 [2026-10-29 17:45:05,746] INFO : Step 39300/50000 rank2 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0803 | Regression_eta: 0.0622 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0806 | LR: 1.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9349s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank2 [2026-10-29 17:49:01,211] INFO : Step 39400/50000 rank2 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0837 | Regression_eta: 0.0490 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0942 | LR: 1.96e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0466s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank2 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank2 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0814 | Regression_eta: 0.0732 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0553 | LR: 1.94e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.2581s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank2 [2026-10-29 17:56:59,044] INFO : Step 39600/50000 rank2 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0863 | Regression_eta: 0.0615 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0784 | LR: 1.93e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1949s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank2 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank2 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0775 | Regression_eta: 0.0797 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0546 | LR: 1.91e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2361s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank2 [2026-10-29 18:04:32,042] INFO : Step 39800/50000 rank2 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0724 | Regression_eta: 0.0493 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0581 | LR: 1.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8998s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank2 [2026-10-29 18:08:19,424] INFO : Step 39900/50000 rank2 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0868 | Regression_eta: 0.0481 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0839 | LR: 1.88e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1819s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank2 [2026-10-29 18:12:05,433] INFO : Step 40000/50000 rank2 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0824 | Regression_eta: 0.0496 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0682 | LR: 1.86e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.9402s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank2 [2026-10-29 18:12:06,327] INFO : Running validation on rank2 +rank2 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:12:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:12:59,647] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 18:13:23,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:13:24,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:13:25,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:13:26,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:13:36,544] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:13:37,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:13:37,722] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 18:13:53,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:13:54,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:13:55,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:13:56,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:14:06,620] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:14:07,598] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 18:14:22,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:14:23,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:14:24,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:14:24,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:14:34,822] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 18:18:48,523] INFO : Step 40100/50000 rank2 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0841 | Regression_eta: 0.0496 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0903 | LR: 1.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1848s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank2 [2026-10-29 18:22:51,411] INFO : Step 40200/50000 rank2 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0800 | Regression_eta: 0.0526 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0752 | LR: 1.83e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.9271s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank2 [2026-10-29 18:26:53,013] INFO : Step 40300/50000 rank2 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0855 | Regression_eta: 0.0509 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0731 | LR: 1.81e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4869s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank2 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank2 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0754 | Regression_eta: 0.0631 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0694 | LR: 1.79e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0130s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank2 [2026-10-29 18:34:31,872] INFO : Step 40500/50000 rank2 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1746 | Regression_cos_phi: 0.1144 | Regression_energy: 0.0918 | Regression_eta: 0.0640 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0937 | LR: 1.78e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.4813s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank2 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank2 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2445 | Regression_cos_phi: 0.0816 | Regression_energy: 0.1354 | Regression_eta: 0.0793 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0985 | LR: 1.76e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1590s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank2 [2026-10-29 18:42:09,669] INFO : Step 40700/50000 rank2 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0734 | Regression_eta: 0.0458 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0619 | LR: 1.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5725s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank2 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank2 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0795 | Regression_eta: 0.0457 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0566 | LR: 1.73e-04 | DataLoad Time: 0.0233s | Model Forward Time: 4.0684s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank2 [2026-10-29 18:49:49,169] INFO : Step 40900/50000 rank2 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2265 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0884 | Regression_eta: 0.0570 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0678 | LR: 1.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6414s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank2 [2026-10-29 18:53:41,414] INFO : Step 41000/50000 rank2 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.2124 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0936 | Regression_eta: 0.0615 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0634 | LR: 1.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4690s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank2 [2026-10-29 18:53:42,383] INFO : Running validation on rank2 +rank2 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:54:33,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:54:33,854] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 18:54:59,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:55:00,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:55:01,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:55:02,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:55:12,612] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:55:13,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:55:13,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 18:55:30,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:55:31,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:55:32,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:55:32,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:55:42,937] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 18:55:59,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:55:59,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:56:00,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:56:01,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:56:11,128] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 19:00:33,894] INFO : Step 41100/50000 rank2 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0769 | Regression_eta: 0.0481 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0526 | LR: 1.69e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.8492s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank2 [2026-10-29 19:04:28,229] INFO : Step 41200/50000 rank2 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0973 | Regression_eta: 0.0553 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0617 | LR: 1.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9465s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank2 [2026-10-29 19:08:18,072] INFO : Step 41300/50000 rank2 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2124 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0861 | Regression_eta: 0.0805 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1009 | LR: 1.66e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1402s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank2 [2026-10-29 19:12:02,348] INFO : Step 41400/50000 rank2 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0862 | Regression_eta: 0.0606 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0797 | LR: 1.64e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9388s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank2 [2026-10-29 19:15:48,707] INFO : Step 41500/50000 rank2 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0770 | Regression_eta: 0.0517 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0734 | LR: 1.63e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.3264s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank2 [2026-10-29 19:19:48,125] INFO : Step 41600/50000 rank2 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0919 | Regression_eta: 0.0514 | Regression_pt: 0.0961 | Regression_sin_phi: 0.1028 | LR: 1.61e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7663s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank2 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank2 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0798 | Regression_eta: 0.0417 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0513 | LR: 1.60e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.9491s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank2 [2026-10-29 19:27:38,847] INFO : Step 41800/50000 rank2 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0510 | Regression_energy: 0.1003 | Regression_eta: 0.0694 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0849 | LR: 1.58e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.3163s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank2 [2026-10-29 19:31:32,457] INFO : Step 41900/50000 rank2 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0802 | Regression_eta: 0.0389 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0560 | LR: 1.57e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.4507s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank2 [2026-10-29 19:35:32,891] INFO : Step 42000/50000 rank2 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2277 | Regression_cos_phi: 0.1048 | Regression_energy: 0.0979 | Regression_eta: 0.0714 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0951 | LR: 1.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9415s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank2 [2026-10-29 19:35:33,871] INFO : Running validation on rank2 +rank2 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 19:36:22,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:36:23,272] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 19:36:50,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 19:36:51,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 19:36:52,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 19:36:53,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 19:37:03,064] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:37:04,354] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 19:37:20,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 19:37:21,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 19:37:22,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 19:37:23,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 19:37:33,338] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:37:33,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:37:34,171] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 19:37:49,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 19:37:50,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 19:37:50,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 19:37:51,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 19:38:01,349] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 19:42:30,051] INFO : Step 42100/50000 rank2 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0871 | Regression_eta: 0.0544 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0724 | LR: 1.54e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0544s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank2 [2026-10-29 19:46:36,781] INFO : Step 42200/50000 rank2 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0874 | Regression_eta: 0.0434 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0770 | LR: 1.53e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.8393s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank2 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank2 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 19:50:51,513] INFO : Step 42300/50000 rank2 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1961 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0969 | Regression_eta: 0.0484 | Regression_pt: 0.1013 | Regression_sin_phi: 0.1106 | LR: 1.52e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2031s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank2 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank2 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0962 | Regression_eta: 0.0654 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0738 | LR: 1.50e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.6749s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank2 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank2 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0850 | Regression_energy: 0.1162 | Regression_eta: 0.0723 | Regression_pt: 0.1218 | Regression_sin_phi: 0.1057 | LR: 1.49e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1945s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank2 [2026-10-29 20:02:59,632] INFO : Step 42600/50000 rank2 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2436 | Regression_cos_phi: 0.0920 | Regression_energy: 0.0841 | Regression_eta: 0.0624 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0906 | LR: 1.48e-04 | DataLoad Time: 0.0152s | Model Forward Time: 8.8098s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank2 [2026-10-29 20:06:58,089] INFO : Step 42700/50000 rank2 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0956 | Regression_eta: 0.0583 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0871 | LR: 1.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8004s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank2 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank2 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0587 | Regression_energy: 0.1000 | Regression_eta: 0.0404 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0460 | LR: 1.45e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5858s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank2 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank2 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2574 | Regression_cos_phi: 0.0573 | Regression_energy: 0.1006 | Regression_eta: 0.0650 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0917 | LR: 1.44e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.4271s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank2 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank2 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0738 | Regression_eta: 0.0462 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0616 | LR: 1.43e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9443s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank2 [2026-10-29 20:18:48,627] INFO : Running validation on rank2 +rank2 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 20:19:38,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:19:39,386] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 20:20:09,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 20:20:10,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 20:20:11,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 20:20:12,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 20:20:22,689] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:20:23,798] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 20:20:39,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 20:20:40,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 20:20:41,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 20:20:42,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 20:20:52,512] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:20:53,998] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 20:21:09,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 20:21:09,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 20:21:10,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 20:21:11,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 20:21:21,202] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 20:25:40,525] INFO : Step 43100/50000 rank2 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0903 | Regression_eta: 0.0533 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0668 | LR: 1.42e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.2858s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank2 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank2 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0845 | Regression_eta: 0.0579 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0820 | LR: 1.40e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank2 [2026-10-29 20:33:23,252] INFO : Step 43300/50000 rank2 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2453 | Regression_cos_phi: 0.0820 | Regression_energy: 0.1026 | Regression_eta: 0.0706 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0819 | LR: 1.39e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank2 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank2 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0784 | Regression_eta: 0.0458 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0656 | LR: 1.38e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5480s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank2 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank2 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0772 | Regression_eta: 0.0473 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0631 | LR: 1.37e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1838s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank2 [2026-10-29 20:44:45,321] INFO : Step 43600/50000 rank2 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0740 | Regression_eta: 0.0489 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0617 | LR: 1.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0821s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank2 [2026-10-29 20:48:25,994] INFO : Step 43700/50000 rank2 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0760 | Regression_eta: 0.0608 | Regression_pt: 0.0795 | Regression_sin_phi: 0.1008 | LR: 1.35e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.4610s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank2 [2026-10-29 20:52:14,801] INFO : Step 43800/50000 rank2 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0818 | Regression_eta: 0.0486 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0524 | LR: 1.34e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.5354s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank2 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank2 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0959 | Regression_eta: 0.0648 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0878 | LR: 1.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3479s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank2 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank2 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0890 | Regression_eta: 0.0718 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0812 | LR: 1.32e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1961s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank2 [2026-10-29 21:00:02,713] INFO : Running validation on rank2 +rank2 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:00:57,489] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:00:58,022] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 21:01:24,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:01:25,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:01:26,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:01:27,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:01:37,678] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:01:38,956] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 21:01:54,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:01:55,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:01:56,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:01:57,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:02:07,656] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:02:08,957] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 21:02:24,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:02:25,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:02:25,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:02:26,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:02:36,567] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 21:06:42,682] INFO : Step 44100/50000 rank2 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0757 | Regression_eta: 0.0518 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0503 | LR: 1.31e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.1685s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank2 [2026-10-29 21:10:40,730] INFO : Step 44200/50000 rank2 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0839 | Regression_eta: 0.0680 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0955 | LR: 1.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9703s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank2 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank2 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0828 | Regression_eta: 0.0542 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0781 | LR: 1.29e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1180s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank2 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank2 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1719 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0816 | Regression_eta: 0.0404 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0515 | LR: 1.28e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7687s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank2 [2026-10-29 21:22:16,363] INFO : Step 44500/50000 rank2 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0717 | Regression_eta: 0.0423 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0639 | LR: 1.27e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8093s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank2 [2026-10-29 21:26:10,595] INFO : Step 44600/50000 rank2 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2023 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0899 | Regression_eta: 0.0802 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0905 | LR: 1.26e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.1499s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank2 [2026-10-29 21:30:16,074] INFO : Step 44700/50000 rank2 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0837 | Regression_eta: 0.0371 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0735 | LR: 1.25e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6981s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank2 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank2 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0866 | Regression_eta: 0.0381 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0456 | LR: 1.24e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.4086s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank2 [2026-10-29 21:38:11,006] INFO : Step 44900/50000 rank2 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0897 | Regression_eta: 0.0436 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0527 | LR: 1.23e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9537s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank2 [2026-10-29 21:42:13,646] INFO : Step 45000/50000 rank2 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2002 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0690 | Regression_eta: 0.0639 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0814 | LR: 1.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0117s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank2 [2026-10-29 21:42:14,559] INFO : Running validation on rank2 +rank2 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:43:06,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:43:07,017] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 21:43:32,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:43:33,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:43:34,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:43:35,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:43:45,847] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:43:47,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:43:47,811] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 21:44:03,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:44:04,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:44:05,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:44:06,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:44:16,382] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:44:17,709] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 21:44:33,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:44:33,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:44:34,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:44:35,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:44:45,244] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 21:48:55,616] INFO : Step 45100/50000 rank2 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0839 | Regression_eta: 0.0690 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0921 | LR: 1.21e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4097s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank2 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank2 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0981 | Regression_eta: 0.0625 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0833 | LR: 1.20e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1521s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank2 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank2 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2402 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0973 | Regression_eta: 0.0644 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0837 | LR: 1.19e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.0119s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank2 [2026-10-29 22:00:32,927] INFO : Step 45400/50000 rank2 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.2002 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0764 | Regression_eta: 0.0697 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0716 | LR: 1.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9036s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank2 [2026-10-29 22:04:23,231] INFO : Step 45500/50000 rank2 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0938 | Regression_eta: 0.0492 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0766 | LR: 1.18e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6350s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank2 [2026-10-29 22:08:21,193] INFO : Step 45600/50000 rank2 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0850 | Regression_eta: 0.0455 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0768 | LR: 1.17e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.0178s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank2 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank2 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0754 | Regression_eta: 0.0439 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0605 | LR: 1.16e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.0841s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank2 [2026-10-29 22:16:08,602] INFO : Step 45800/50000 rank2 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0893 | Regression_eta: 0.0375 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0653 | LR: 1.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6678s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank2 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank2 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0711 | Regression_eta: 0.0306 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0480 | LR: 1.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9608s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank2 [2026-10-29 22:24:24,450] INFO : Step 46000/50000 rank2 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0768 | Regression_eta: 0.0597 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0595 | LR: 1.14e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2989s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank2 [2026-10-29 22:24:25,403] INFO : Running validation on rank2 +rank2 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 22:25:16,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:25:17,231] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 22:25:46,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 22:25:47,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 22:25:48,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 22:25:49,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 22:25:59,359] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 22:25:59,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:26:00,549] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 22:26:16,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 22:26:17,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 22:26:18,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 22:26:20,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 22:26:30,222] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:26:31,921] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 22:26:47,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 22:26:47,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 22:26:48,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 22:26:49,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 22:26:59,252] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 22:31:16,604] INFO : Step 46100/50000 rank2 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0721 | Regression_eta: 0.0447 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0582 | LR: 1.13e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0438s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank2 [2026-10-29 22:35:12,810] INFO : Step 46200/50000 rank2 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0839 | Regression_eta: 0.0497 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0593 | LR: 1.13e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9303s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank2 [2026-10-29 22:39:25,150] INFO : Step 46300/50000 rank2 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0777 | Regression_eta: 0.0547 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0634 | LR: 1.12e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.9864s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank2 [2026-10-29 22:43:23,785] INFO : Step 46400/50000 rank2 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0962 | Regression_eta: 0.0845 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0663 | LR: 1.11e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.6669s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank2 [2026-10-29 22:47:17,351] INFO : Step 46500/50000 rank2 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0856 | Regression_eta: 0.0608 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0873 | LR: 1.11e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.5736s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank2 [2026-10-29 22:51:20,583] INFO : Step 46600/50000 rank2 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0732 | Regression_eta: 0.0399 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0495 | LR: 1.10e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3964s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank2 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank2 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0840 | Regression_eta: 0.0466 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0794 | LR: 1.10e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0460s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank2 [2026-10-29 22:59:26,795] INFO : Step 46800/50000 rank2 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0942 | Regression_eta: 0.0371 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0576 | LR: 1.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9109s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank2 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank2 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0896 | Regression_eta: 0.0600 | Regression_pt: 0.0949 | Regression_sin_phi: 0.1163 | LR: 1.09e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.0396s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank2 [2026-10-29 23:08:15,439] INFO : Step 47000/50000 rank2 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2230 | Regression_cos_phi: 0.1061 | Regression_energy: 0.0745 | Regression_eta: 0.0711 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0867 | LR: 1.08e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2714s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank2 [2026-10-29 23:08:16,507] INFO : Running validation on rank2 +rank2 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 23:09:07,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:09:08,184] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 23:09:36,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 23:09:38,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 23:09:40,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 23:09:42,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 23:09:52,739] INFO : Time taken to make predictions on device 2 is: 0.32 min +rank2 [2026-10-29 23:09:55,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:09:55,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 23:10:11,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 23:10:12,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 23:10:13,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 23:10:14,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 23:10:24,474] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:10:25,936] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 23:10:41,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 23:10:42,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 23:10:42,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 23:10:43,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 23:10:53,400] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 23:15:13,932] INFO : Step 47100/50000 rank2 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0945 | Regression_eta: 0.0516 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0615 | LR: 1.07e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7042s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank2 [2026-10-29 23:19:19,773] INFO : Step 47200/50000 rank2 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0627 | Regression_eta: 0.0493 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0526 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.6701s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank2 [2026-10-29 23:23:31,493] INFO : Step 47300/50000 rank2 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0925 | Regression_eta: 0.0709 | Regression_pt: 0.0975 | Regression_sin_phi: 0.1030 | LR: 1.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7158s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank2 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank2 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0789 | Regression_eta: 0.0680 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0518 | LR: 1.06e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0196s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank2 [2026-10-29 23:31:38,583] INFO : Step 47500/50000 rank2 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0893 | Regression_eta: 0.0554 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0594 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8380s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank2 [2026-10-29 23:35:34,355] INFO : Step 47600/50000 rank2 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2510 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1014 | Regression_eta: 0.0542 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0983 | LR: 1.05e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5026s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank2 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank2 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0734 | Regression_eta: 0.0474 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0442 | LR: 1.05e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8401s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank2 [2026-10-29 23:43:53,970] INFO : Step 47800/50000 rank2 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0760 | Regression_eta: 0.0424 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0817 | LR: 1.04e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6124s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank2 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank2 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0949 | Regression_eta: 0.0544 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0600 | LR: 1.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1959s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank2 [2026-10-29 23:51:38,726] INFO : Step 48000/50000 rank2 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0792 | Regression_eta: 0.0541 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0654 | LR: 1.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2963s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank2 [2026-10-29 23:51:39,732] INFO : Running validation on rank2 +rank2 [2026-10-29 23:51:39,732] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 23:51:39,732] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 23:52:31,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:52:31,695] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 23:52:58,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 23:52:59,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 23:53:01,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 23:53:02,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 23:53:12,970] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-29 23:53:13,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 23:53:29,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 23:53:30,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 23:53:31,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 23:53:32,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 23:53:42,514] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:53:43,871] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 23:53:58,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 23:53:59,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 23:54:00,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 23:54:00,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 23:54:10,852] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-30 00:00:53,531] INFO : Step 48100/50000 rank2 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0698 | Regression_eta: 0.0573 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0449 | LR: 1.03e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5289s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank2 [2026-10-30 00:04:40,584] INFO : Step 48200/50000 rank2 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0819 | Regression_eta: 0.0709 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0805 | LR: 1.03e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.0265s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank2 [2026-10-30 00:08:31,628] INFO : Step 48300/50000 rank2 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2054 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0791 | Regression_eta: 0.0427 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0714 | LR: 1.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9448s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank2 [2026-10-30 00:12:19,575] INFO : Step 48400/50000 rank2 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2319 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0853 | Regression_eta: 0.0729 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0869 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3347s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank2 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank2 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0748 | Regression_eta: 0.0461 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0627 | LR: 1.02e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.5876s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank2 [2026-10-30 00:19:50,480] INFO : Step 48600/50000 rank2 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0826 | Regression_eta: 0.0594 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0714 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9683s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank2 [2026-10-30 00:23:38,553] INFO : Step 48700/50000 rank2 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0703 | Regression_eta: 0.0426 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0642 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0931s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank2 [2026-10-30 00:27:30,349] INFO : Step 48800/50000 rank2 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0828 | Regression_eta: 0.0587 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0614 | LR: 1.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4235s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank2 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank2 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0796 | Regression_eta: 0.0705 | Regression_pt: 0.0821 | Regression_sin_phi: 0.1346 | LR: 1.01e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6425s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank2 [2026-10-30 00:35:14,112] INFO : Step 49000/50000 rank2 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0750 | Regression_eta: 0.0451 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0656 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5976s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank2 [2026-10-30 00:35:15,047] INFO : Running validation on rank2 +rank2 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:36:03,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 00:36:29,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:36:31,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:36:32,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:36:33,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:36:43,447] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-30 00:36:48,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:36:48,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 00:37:04,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:37:05,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:37:06,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:37:07,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:37:17,370] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:37:18,486] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 00:37:33,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 00:37:34,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 00:37:34,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 00:37:35,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 00:37:45,515] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-30 00:41:49,929] INFO : Step 49100/50000 rank2 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0790 | Regression_eta: 0.0546 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0823 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6165s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank2 [2026-10-30 00:45:59,116] INFO : Step 49200/50000 rank2 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0714 | Regression_eta: 0.0417 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0645 | LR: 1.01e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.5925s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank2 [2026-10-30 00:49:54,419] INFO : Step 49300/50000 rank2 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2200 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0782 | Regression_eta: 0.0926 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0826 | LR: 1.00e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9870s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank2 [2026-10-30 00:53:44,254] INFO : Step 49400/50000 rank2 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0935 | Regression_eta: 0.0384 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0617 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9615s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank2 [2026-10-30 00:57:39,409] INFO : Step 49500/50000 rank2 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0846 | Regression_eta: 0.0605 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0505 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0124s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank2 [2026-10-30 01:01:39,141] INFO : Step 49600/50000 rank2 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0661 | Regression_eta: 0.0419 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0607 | LR: 1.00e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8029s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank2 [2026-10-30 01:05:30,842] INFO : Step 49700/50000 rank2 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0784 | Regression_eta: 0.0499 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0723 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4627s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank2 [2026-10-30 01:09:21,933] INFO : Step 49800/50000 rank2 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0761 | Regression_eta: 0.0645 | Regression_pt: 0.0798 | Regression_sin_phi: 0.1099 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3599s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank2 [2026-10-30 01:13:13,492] INFO : Step 49900/50000 rank2 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0909 | Regression_eta: 0.0651 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0720 | LR: 1.00e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0824s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank2 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank2 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.2026 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0735 | Regression_eta: 0.0554 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0734 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8272s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank2 [2026-10-30 01:17:09,619] INFO : Running validation on rank2 +rank2 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 01:18:02,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:18:03,255] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 01:18:29,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 01:18:30,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 01:18:31,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 01:18:32,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 01:18:42,059] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:18:42,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:18:43,092] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 01:18:59,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 01:19:00,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 01:19:01,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 01:19:01,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 01:19:11,920] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 01:19:28,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:19:29,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:19:30,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:19:30,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:19:40,808] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-30 01:19:41,619] INFO : Training completed. Total time on device 2: 629.366min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.3 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..6d90a24dfbe34bfc7656b3a203c7ce7f84d28492 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.3 @@ -0,0 +1,2483 @@ +rank3 [2026-10-28 11:12:32,339] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-28 11:12:32,340] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank3 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-28 11:12:57,453] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-28 11:12:57,453] INFO : nn0 parameters: 171776 +rank3 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank3 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 1 +rank3 [2026-10-28 11:12:57,459] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,464] INFO : Initializing backbone layer 2 +rank3 [2026-10-28 11:12:57,464] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,468] INFO : Initializing backbone layer 3 +rank3 [2026-10-28 11:12:57,468] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,471] INFO : Initializing backbone layer 4 +rank3 [2026-10-28 11:12:57,471] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,474] INFO : Initializing backbone layer 5 +rank3 [2026-10-28 11:12:57,475] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:57,478] INFO : Backbone initialization took 0.02s +rank3 [2026-10-28 11:12:57,478] INFO : backbone parameters: 2374656 +rank3 [2026-10-28 11:12:57,478] INFO : Initializing output DNNs +rank3 [2026-10-28 11:12:57,522] INFO : Output DNNs initialization took 0.04s +rank3 [2026-10-28 11:12:57,522] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-28 11:12:57,522] INFO : jagged_attention enabled=True +rank3 [2026-10-28 11:12:57,522] INFO : flash_attn_varlen enabled=True +rank3 [2026-10-28 11:12:57,523] INFO : task_queries enabled=False +rank3 [2026-10-28 11:12:57,523] INFO : classification_norm parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : regression_norm parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : classification_query parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank3 [2026-10-28 11:12:57,523] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-28 11:12:57,523] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : nn_pid parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : nn_pt parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : nn_eta parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-28 11:12:57,523] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-28 11:12:57,524] INFO : nn_energy parameters: 0 +rank3 [2026-10-28 11:12:57,524] INFO : Total MLPF parameters: 7260244 +rank3 [2026-10-28 11:12:57,524] INFO : MLPF __init__ done +rank3 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank3 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=3 +rank3 [2026-10-28 11:12:57,557] INFO : Moved model to device rank=3 +rank3 [2026-10-28 11:12:57,559] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-28 11:12:57,559] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-28 11:12:57,813] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank3 [2026-10-28 11:12:57,813] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-28 11:12:57,814] INFO : Trainable parameters: 7260244 +rank3 [2026-10-28 11:12:57,814] INFO : Non-trainable parameters: 0 +rank3 [2026-10-28 11:12:57,814] INFO : Total parameters: 7260244 +rank3 [2026-10-28 11:12:57,817] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-28 11:12:57,817] INFO : Getting dataloaders +rank3 [2026-10-28 11:12:57,817] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:03,710] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:04,771] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,002] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,283] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,560] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,823] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-28 11:13:06,823] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-28 11:13:20,914] INFO : Got dataloaders +rank3 [2026-10-28 11:13:20,914] INFO : loader split=train rank=3 len=21108 +rank3 [2026-10-28 11:13:20,914] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-28 11:13:20,914] INFO : Starting training from step 1 to 50000 on rank 3 of 8 +rank3 [2026-10-28 11:13:20,914] INFO : Creating train_iterator +rank3 [2026-10-28 11:13:20,914] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-28 11:13:35,221] INFO : Created train_iterator +rank3 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank3 [2026-10-28 11:17:20,490] INFO : Step 100/50000 rank3 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8398 | Classification_binary: 0.4627 | Regression_cos_phi: 0.0068 | Regression_energy: 1.0901 | Regression_eta: 0.0205 | Regression_pt: 1.0402 | Regression_sin_phi: 0.0086 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 3.1319s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank3 [2026-10-28 11:20:57,690] INFO : Step 200/50000 rank3 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4653 | Classification_binary: 0.4976 | Regression_cos_phi: 0.4257 | Regression_energy: 0.3528 | Regression_eta: 0.3632 | Regression_pt: 0.3790 | Regression_sin_phi: 0.3737 | LR: 1.00e-03 | DataLoad Time: 0.0151s | Model Forward Time: 3.5615s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank3 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank3 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2833 | Classification_binary: 0.4931 | Regression_cos_phi: 0.3051 | Regression_energy: 0.2970 | Regression_eta: 0.2676 | Regression_pt: 0.3030 | Regression_sin_phi: 0.3042 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 3.7567s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank3 [2026-10-28 11:28:09,552] INFO : Step 400/50000 rank3 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2924 | Classification_binary: 0.4075 | Regression_cos_phi: 0.2909 | Regression_energy: 0.2649 | Regression_eta: 0.2515 | Regression_pt: 0.2760 | Regression_sin_phi: 0.2100 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 3.5811s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank3 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank3 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3385 | Classification_binary: 0.4904 | Regression_cos_phi: 0.3107 | Regression_energy: 0.3164 | Regression_eta: 0.2544 | Regression_pt: 0.3233 | Regression_sin_phi: 0.2784 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 4.0269s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank3 [2026-10-28 11:35:32,723] INFO : Step 600/50000 rank3 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2439 | Classification_binary: 0.4894 | Regression_cos_phi: 0.2844 | Regression_energy: 0.2649 | Regression_eta: 0.2399 | Regression_pt: 0.2824 | Regression_sin_phi: 0.2840 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.1152s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank3 [2026-10-28 11:39:09,491] INFO : Step 700/50000 rank3 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.4067 | Regression_cos_phi: 0.2998 | Regression_energy: 0.2525 | Regression_eta: 0.3057 | Regression_pt: 0.2672 | Regression_sin_phi: 0.2676 | LR: 1.00e-03 | DataLoad Time: 0.0172s | Model Forward Time: 3.3984s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank3 [2026-10-28 11:42:49,057] INFO : Step 800/50000 rank3 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2547 | Classification_binary: 0.4881 | Regression_cos_phi: 0.4007 | Regression_energy: 0.3116 | Regression_eta: 0.3222 | Regression_pt: 0.3374 | Regression_sin_phi: 0.3867 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.6499s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank3 [2026-10-28 11:46:31,921] INFO : Step 900/50000 rank3 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2766 | Classification_binary: 0.5325 | Regression_cos_phi: 0.3482 | Regression_energy: 0.3065 | Regression_eta: 0.3040 | Regression_pt: 0.3203 | Regression_sin_phi: 0.3016 | LR: 9.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5688s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank3 [2026-10-28 11:50:09,950] INFO : Step 1000/50000 rank3 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2346 | Classification_binary: 0.4396 | Regression_cos_phi: 0.1947 | Regression_energy: 0.3060 | Regression_eta: 0.2260 | Regression_pt: 0.3270 | Regression_sin_phi: 0.2427 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4174s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank3 [2026-10-28 11:50:10,910] INFO : Running validation on rank3 +rank3 [2026-10-28 11:50:10,910] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 11:50:58,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:50:59,089] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 11:51:19,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 11:51:20,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 11:51:21,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 11:51:22,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 11:51:32,317] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 11:51:32,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:51:33,312] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 11:51:49,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 11:51:50,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 11:51:51,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 11:51:52,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 11:52:02,064] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 11:52:02,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:52:03,469] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 11:52:18,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 11:52:19,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 11:52:20,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 11:52:20,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 11:52:31,005] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 11:56:43,807] INFO : Step 1100/50000 rank3 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2100 | Classification_binary: 0.4219 | Regression_cos_phi: 0.3210 | Regression_energy: 0.2991 | Regression_eta: 0.2470 | Regression_pt: 0.3095 | Regression_sin_phi: 0.2516 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9238s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank3 [2026-10-28 12:00:30,891] INFO : Step 1200/50000 rank3 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2285 | Classification_binary: 0.5141 | Regression_cos_phi: 0.3186 | Regression_energy: 0.2846 | Regression_eta: 0.2986 | Regression_pt: 0.2975 | Regression_sin_phi: 0.2913 | LR: 9.99e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3993s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank3 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank3 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1924 | Classification_binary: 0.4143 | Regression_cos_phi: 0.3140 | Regression_energy: 0.3149 | Regression_eta: 0.2696 | Regression_pt: 0.3299 | Regression_sin_phi: 0.1997 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1273s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank3 [2026-10-28 12:08:10,852] INFO : Step 1400/50000 rank3 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2400 | Classification_binary: 0.4091 | Regression_cos_phi: 0.2853 | Regression_energy: 0.2469 | Regression_eta: 0.2161 | Regression_pt: 0.2574 | Regression_sin_phi: 0.1932 | LR: 9.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7556s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank3 [2026-10-28 12:11:57,638] INFO : Step 1500/50000 rank3 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2379 | Classification_binary: 0.4323 | Regression_cos_phi: 0.2486 | Regression_energy: 0.2739 | Regression_eta: 0.2115 | Regression_pt: 0.2843 | Regression_sin_phi: 0.2264 | LR: 9.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5797s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank3 [2026-10-28 12:15:43,591] INFO : Step 1600/50000 rank3 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2152 | Classification_binary: 0.4431 | Regression_cos_phi: 0.2521 | Regression_energy: 0.2731 | Regression_eta: 0.2009 | Regression_pt: 0.2874 | Regression_sin_phi: 0.2496 | LR: 9.98e-04 | DataLoad Time: 0.0249s | Model Forward Time: 4.0347s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank3 [2026-10-28 12:19:37,262] INFO : Step 1700/50000 rank3 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1955 | Classification_binary: 0.3891 | Regression_cos_phi: 0.2407 | Regression_energy: 0.2462 | Regression_eta: 0.2388 | Regression_pt: 0.2585 | Regression_sin_phi: 0.2439 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.3922s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank3 [2026-10-28 12:23:17,460] INFO : Step 1800/50000 rank3 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2085 | Classification_binary: 0.4190 | Regression_cos_phi: 0.2970 | Regression_energy: 0.2325 | Regression_eta: 0.2584 | Regression_pt: 0.2435 | Regression_sin_phi: 0.2539 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9417s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank3 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank3 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2262 | Classification_binary: 0.4831 | Regression_cos_phi: 0.2346 | Regression_energy: 0.2469 | Regression_eta: 0.1968 | Regression_pt: 0.2573 | Regression_sin_phi: 0.2135 | LR: 9.97e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.5720s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank3 [2026-10-28 12:30:59,047] INFO : Step 2000/50000 rank3 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1720 | Classification_binary: 0.3970 | Regression_cos_phi: 0.2616 | Regression_energy: 0.2139 | Regression_eta: 0.2107 | Regression_pt: 0.2249 | Regression_sin_phi: 0.2536 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1427s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank3 [2026-10-28 12:31:00,063] INFO : Running validation on rank3 +rank3 [2026-10-28 12:31:00,063] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 12:31:47,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:31:48,296] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 12:32:13,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:32:14,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:32:15,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:32:16,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:32:26,424] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:32:27,680] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 12:32:43,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:32:44,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:32:45,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:32:46,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:32:56,541] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:32:57,119] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:32:57,645] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 12:33:13,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:33:14,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:33:15,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:33:16,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:33:26,157] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:37:32,605] INFO : Step 2100/50000 rank3 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1935 | Classification_binary: 0.3618 | Regression_cos_phi: 0.2166 | Regression_energy: 0.2699 | Regression_eta: 0.2248 | Regression_pt: 0.2841 | Regression_sin_phi: 0.2373 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7725s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank3 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank3 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1819 | Classification_binary: 0.4256 | Regression_cos_phi: 0.2380 | Regression_energy: 0.2658 | Regression_eta: 0.2249 | Regression_pt: 0.2801 | Regression_sin_phi: 0.2215 | LR: 9.96e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7062s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank3 [2026-10-28 12:45:14,506] INFO : Step 2300/50000 rank3 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2645 | Classification_binary: 0.4286 | Regression_cos_phi: 0.2703 | Regression_energy: 0.2279 | Regression_eta: 0.2298 | Regression_pt: 0.2389 | Regression_sin_phi: 0.2705 | LR: 9.95e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9941s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank3 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank3 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1821 | Classification_binary: 0.3138 | Regression_cos_phi: 0.2722 | Regression_energy: 0.2496 | Regression_eta: 0.2911 | Regression_pt: 0.2587 | Regression_sin_phi: 0.2678 | LR: 9.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4488s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank3 [2026-10-28 12:52:58,132] INFO : Step 2500/50000 rank3 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2383 | Classification_binary: 0.4218 | Regression_cos_phi: 0.2363 | Regression_energy: 0.2509 | Regression_eta: 0.1898 | Regression_pt: 0.2623 | Regression_sin_phi: 0.2309 | LR: 9.94e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3992s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank3 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank3 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1888 | Classification_binary: 0.4146 | Regression_cos_phi: 0.2509 | Regression_energy: 0.2551 | Regression_eta: 0.1839 | Regression_pt: 0.2649 | Regression_sin_phi: 0.1963 | LR: 9.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank3 [2026-10-28 13:00:38,243] INFO : Step 2700/50000 rank3 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1748 | Classification_binary: 0.3431 | Regression_cos_phi: 0.2082 | Regression_energy: 0.2359 | Regression_eta: 0.1800 | Regression_pt: 0.2492 | Regression_sin_phi: 0.2939 | LR: 9.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0880s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank3 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank3 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1643 | Classification_binary: 0.3464 | Regression_cos_phi: 0.2814 | Regression_energy: 0.2798 | Regression_eta: 0.1949 | Regression_pt: 0.2944 | Regression_sin_phi: 0.2363 | LR: 9.93e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1946s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank3 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank3 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2051 | Classification_binary: 0.4555 | Regression_cos_phi: 0.2533 | Regression_energy: 0.2442 | Regression_eta: 0.2333 | Regression_pt: 0.2561 | Regression_sin_phi: 0.2473 | LR: 9.93e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1946s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank3 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank3 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1693 | Classification_binary: 0.3486 | Regression_cos_phi: 0.2166 | Regression_energy: 0.2156 | Regression_eta: 0.1887 | Regression_pt: 0.2223 | Regression_sin_phi: 0.1626 | LR: 9.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5278s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank3 [2026-10-28 13:12:10,495] INFO : Running validation on rank3 +rank3 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:13:02,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:13:03,432] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 13:13:30,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:13:30,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:13:31,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:13:32,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:13:42,870] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:13:44,104] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 13:14:00,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:14:01,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:14:02,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:14:02,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:14:12,989] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:14:14,292] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 13:14:29,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:14:30,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:14:31,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:14:32,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:14:42,010] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 13:19:02,742] INFO : Step 3100/50000 rank3 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1755 | Classification_binary: 0.3194 | Regression_cos_phi: 0.1757 | Regression_energy: 0.2167 | Regression_eta: 0.1473 | Regression_pt: 0.2280 | Regression_sin_phi: 0.2132 | LR: 9.91e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.5829s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank3 [2026-10-28 13:23:00,673] INFO : Step 3200/50000 rank3 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2196 | Classification_binary: 0.4425 | Regression_cos_phi: 0.3217 | Regression_energy: 0.2404 | Regression_eta: 0.2561 | Regression_pt: 0.2515 | Regression_sin_phi: 0.2858 | LR: 9.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8524s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank3 [2026-10-28 13:26:54,112] INFO : Step 3300/50000 rank3 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1763 | Classification_binary: 0.4042 | Regression_cos_phi: 0.1820 | Regression_energy: 0.2251 | Regression_eta: 0.1984 | Regression_pt: 0.2306 | Regression_sin_phi: 0.2469 | LR: 9.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7317s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank3 [2026-10-28 13:30:48,597] INFO : Step 3400/50000 rank3 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1651 | Classification_binary: 0.3402 | Regression_cos_phi: 0.2392 | Regression_energy: 0.2137 | Regression_eta: 0.2459 | Regression_pt: 0.2165 | Regression_sin_phi: 0.2052 | LR: 9.90e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0803s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank3 [2026-10-28 13:34:47,770] INFO : Step 3500/50000 rank3 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1598 | Classification_binary: 0.3535 | Regression_cos_phi: 0.2572 | Regression_energy: 0.2155 | Regression_eta: 0.1443 | Regression_pt: 0.2312 | Regression_sin_phi: 0.2139 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6675s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank3 [2026-10-28 13:38:46,122] INFO : Step 3600/50000 rank3 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.4382 | Regression_cos_phi: 0.2316 | Regression_energy: 0.1933 | Regression_eta: 0.2013 | Regression_pt: 0.2017 | Regression_sin_phi: 0.1883 | LR: 9.89e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.1038s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank3 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank3 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1656 | Classification_binary: 0.2990 | Regression_cos_phi: 0.2063 | Regression_energy: 0.2466 | Regression_eta: 0.2540 | Regression_pt: 0.2509 | Regression_sin_phi: 0.1978 | LR: 9.88e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2050s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank3 [2026-10-28 13:46:34,310] INFO : Step 3800/50000 rank3 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1683 | Classification_binary: 0.4090 | Regression_cos_phi: 0.2665 | Regression_energy: 0.2012 | Regression_eta: 0.2524 | Regression_pt: 0.2128 | Regression_sin_phi: 0.2209 | LR: 9.87e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9858s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank3 [2026-10-28 13:50:23,156] INFO : Step 3900/50000 rank3 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1569 | Classification_binary: 0.4217 | Regression_cos_phi: 0.2001 | Regression_energy: 0.2106 | Regression_eta: 0.1691 | Regression_pt: 0.2192 | Regression_sin_phi: 0.1708 | LR: 9.87e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2419s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank3 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank3 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 0.3832 | Regression_cos_phi: 0.1459 | Regression_energy: 0.2506 | Regression_eta: 0.1882 | Regression_pt: 0.2644 | Regression_sin_phi: 0.2188 | LR: 9.86e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5790s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank3 [2026-10-28 13:54:12,843] INFO : Running validation on rank3 +rank3 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:55:03,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 13:55:33,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:55:34,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:55:35,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:55:35,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:55:45,941] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:55:46,898] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 13:56:03,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:56:04,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:56:05,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:56:06,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:56:16,125] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:56:16,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:56:16,986] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 13:56:32,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:56:33,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:56:33,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:56:34,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:56:44,593] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank3 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1531 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1700 | Regression_energy: 0.2149 | Regression_eta: 0.1435 | Regression_pt: 0.2214 | Regression_sin_phi: 0.1570 | LR: 9.85e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9529s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank3 [2026-10-28 14:04:50,729] INFO : Step 4200/50000 rank3 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1582 | Classification_binary: 0.3553 | Regression_cos_phi: 0.2179 | Regression_energy: 0.2346 | Regression_eta: 0.1540 | Regression_pt: 0.2432 | Regression_sin_phi: 0.1985 | LR: 9.84e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4866s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank3 [2026-10-28 14:08:42,269] INFO : Step 4300/50000 rank3 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1673 | Classification_binary: 0.3673 | Regression_cos_phi: 0.1947 | Regression_energy: 0.1880 | Regression_eta: 0.1789 | Regression_pt: 0.1978 | Regression_sin_phi: 0.2558 | LR: 9.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.1086s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank3 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank3 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1433 | Classification_binary: 0.3032 | Regression_cos_phi: 0.1745 | Regression_energy: 0.2046 | Regression_eta: 0.1579 | Regression_pt: 0.2125 | Regression_sin_phi: 0.1903 | LR: 9.83e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.7766s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank3 [2026-10-28 14:16:26,827] INFO : Step 4500/50000 rank3 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1642 | Classification_binary: 0.3749 | Regression_cos_phi: 0.1640 | Regression_energy: 0.1856 | Regression_eta: 0.1772 | Regression_pt: 0.1948 | Regression_sin_phi: 0.2233 | LR: 9.82e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4671s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank3 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank3 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1779 | Classification_binary: 0.4357 | Regression_cos_phi: 0.2345 | Regression_energy: 0.2183 | Regression_eta: 0.1965 | Regression_pt: 0.2291 | Regression_sin_phi: 0.2140 | LR: 9.81e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1372s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank3 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank3 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1156 | Classification_binary: 0.3039 | Regression_cos_phi: 0.1586 | Regression_energy: 0.1683 | Regression_eta: 0.1478 | Regression_pt: 0.1763 | Regression_sin_phi: 0.1505 | LR: 9.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3023s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank3 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank3 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1704 | Classification_binary: 0.3633 | Regression_cos_phi: 0.2813 | Regression_energy: 0.2201 | Regression_eta: 0.2380 | Regression_pt: 0.2305 | Regression_sin_phi: 0.2906 | LR: 9.80e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9655s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank3 [2026-10-28 14:32:01,141] INFO : Step 4900/50000 rank3 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1533 | Classification_binary: 0.4172 | Regression_cos_phi: 0.1929 | Regression_energy: 0.1827 | Regression_eta: 0.1713 | Regression_pt: 0.1948 | Regression_sin_phi: 0.2108 | LR: 9.79e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8472s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank3 [2026-10-28 14:35:52,796] INFO : Step 5000/50000 rank3 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1700 | Classification_binary: 0.3859 | Regression_cos_phi: 0.2326 | Regression_energy: 0.1980 | Regression_eta: 0.1879 | Regression_pt: 0.2090 | Regression_sin_phi: 0.1863 | LR: 9.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8409s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank3 [2026-10-28 14:35:53,776] INFO : Running validation on rank3 +rank3 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 14:36:42,157] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:36:42,806] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 14:37:08,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 14:37:09,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 14:37:10,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 14:37:11,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 14:37:21,823] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:37:22,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 14:37:38,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 14:37:39,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 14:37:40,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 14:37:41,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 14:37:51,359] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 14:37:52,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:37:52,790] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 14:38:08,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 14:38:08,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 14:38:09,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 14:38:10,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 14:38:20,429] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank3 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1433 | Classification_binary: 0.3074 | Regression_cos_phi: 0.1813 | Regression_energy: 0.2085 | Regression_eta: 0.1714 | Regression_pt: 0.2175 | Regression_sin_phi: 0.1611 | LR: 9.77e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3393s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank3 [2026-10-28 14:46:36,781] INFO : Step 5200/50000 rank3 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1674 | Classification_binary: 0.4036 | Regression_cos_phi: 0.2030 | Regression_energy: 0.1903 | Regression_eta: 0.1898 | Regression_pt: 0.1996 | Regression_sin_phi: 0.2431 | LR: 9.76e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9490s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank3 [2026-10-28 14:50:33,437] INFO : Step 5300/50000 rank3 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1507 | Classification_binary: 0.3637 | Regression_cos_phi: 0.2439 | Regression_energy: 0.2150 | Regression_eta: 0.1666 | Regression_pt: 0.2230 | Regression_sin_phi: 0.1728 | LR: 9.75e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.1864s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank3 [2026-10-28 14:54:36,491] INFO : Step 5400/50000 rank3 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0988 | Classification_binary: 0.2676 | Regression_cos_phi: 0.1598 | Regression_energy: 0.1683 | Regression_eta: 0.1217 | Regression_pt: 0.1770 | Regression_sin_phi: 0.1491 | LR: 9.74e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4920s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank3 [2026-10-28 14:58:42,458] INFO : Step 5500/50000 rank3 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1707 | Classification_binary: 0.3221 | Regression_cos_phi: 0.2061 | Regression_energy: 0.1907 | Regression_eta: 0.1392 | Regression_pt: 0.2011 | Regression_sin_phi: 0.2123 | LR: 9.73e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.5909s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank3 [2026-10-28 15:02:44,093] INFO : Step 5600/50000 rank3 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1645 | Classification_binary: 0.4061 | Regression_cos_phi: 0.1972 | Regression_energy: 0.1841 | Regression_eta: 0.1987 | Regression_pt: 0.1942 | Regression_sin_phi: 0.2037 | LR: 9.72e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6395s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank3 [2026-10-28 15:06:32,821] INFO : Step 5700/50000 rank3 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1442 | Classification_binary: 0.2880 | Regression_cos_phi: 0.1148 | Regression_energy: 0.2056 | Regression_eta: 0.1352 | Regression_pt: 0.2168 | Regression_sin_phi: 0.1634 | LR: 9.71e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.1511s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank3 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank3 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1419 | Classification_binary: 0.3156 | Regression_cos_phi: 0.1900 | Regression_energy: 0.1754 | Regression_eta: 0.1429 | Regression_pt: 0.1826 | Regression_sin_phi: 0.1941 | LR: 9.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5379s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank3 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank3 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1366 | Classification_binary: 0.3621 | Regression_cos_phi: 0.1889 | Regression_energy: 0.1859 | Regression_eta: 0.1692 | Regression_pt: 0.1932 | Regression_sin_phi: 0.2060 | LR: 9.69e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.0728s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank3 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank3 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1486 | Classification_binary: 0.3062 | Regression_cos_phi: 0.1252 | Regression_energy: 0.1847 | Regression_eta: 0.1170 | Regression_pt: 0.1906 | Regression_sin_phi: 0.1469 | LR: 9.68e-04 | DataLoad Time: 0.0139s | Model Forward Time: 6.3976s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank3 [2026-10-28 15:18:46,643] INFO : Running validation on rank3 +rank3 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:19:35,272] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 15:19:55,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 15:19:57,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 15:19:58,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 15:19:59,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 15:20:09,910] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:20:11,685] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 15:20:27,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 15:20:28,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 15:20:29,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 15:20:30,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 15:20:40,537] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 15:20:40,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:20:41,483] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 15:20:57,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 15:20:58,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 15:20:58,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 15:20:59,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 15:21:09,566] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 15:25:37,243] INFO : Step 6100/50000 rank3 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1404 | Classification_binary: 0.2867 | Regression_cos_phi: 0.1937 | Regression_energy: 0.1787 | Regression_eta: 0.1765 | Regression_pt: 0.1821 | Regression_sin_phi: 0.1263 | LR: 9.67e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2782s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank3 [2026-10-28 15:29:27,138] INFO : Step 6200/50000 rank3 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1705 | Classification_binary: 0.3418 | Regression_cos_phi: 0.2104 | Regression_energy: 0.2031 | Regression_eta: 0.2136 | Regression_pt: 0.2150 | Regression_sin_phi: 0.2649 | LR: 9.66e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.3984s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank3 [2026-10-28 15:33:16,838] INFO : Step 6300/50000 rank3 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1641 | Classification_binary: 0.3931 | Regression_cos_phi: 0.1887 | Regression_energy: 0.1688 | Regression_eta: 0.1669 | Regression_pt: 0.1758 | Regression_sin_phi: 0.2109 | LR: 9.65e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2470s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank3 [2026-10-28 15:37:09,470] INFO : Step 6400/50000 rank3 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1211 | Classification_binary: 0.2912 | Regression_cos_phi: 0.2024 | Regression_energy: 0.1593 | Regression_eta: 0.1444 | Regression_pt: 0.1661 | Regression_sin_phi: 0.1652 | LR: 9.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9978s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank3 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank3 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1001 | Classification_binary: 0.2953 | Regression_cos_phi: 0.1189 | Regression_energy: 0.1633 | Regression_eta: 0.1209 | Regression_pt: 0.1712 | Regression_sin_phi: 0.1479 | LR: 9.63e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6837s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank3 [2026-10-28 15:45:09,041] INFO : Step 6600/50000 rank3 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1268 | Classification_binary: 0.3826 | Regression_cos_phi: 0.1383 | Regression_energy: 0.1796 | Regression_eta: 0.1351 | Regression_pt: 0.1884 | Regression_sin_phi: 0.1765 | LR: 9.62e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2537s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank3 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank3 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1477 | Classification_binary: 0.3487 | Regression_cos_phi: 0.1748 | Regression_energy: 0.1771 | Regression_eta: 0.1761 | Regression_pt: 0.1868 | Regression_sin_phi: 0.1987 | LR: 9.61e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.0307s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank3 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank3 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1164 | Classification_binary: 0.2941 | Regression_cos_phi: 0.1842 | Regression_energy: 0.1608 | Regression_eta: 0.1461 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1188 | LR: 9.60e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8860s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank3 [2026-10-28 15:56:38,728] INFO : Step 6900/50000 rank3 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1537 | Classification_binary: 0.3077 | Regression_cos_phi: 0.1942 | Regression_energy: 0.1807 | Regression_eta: 0.1358 | Regression_pt: 0.1899 | Regression_sin_phi: 0.2052 | LR: 9.58e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8822s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank3 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank3 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1634 | Classification_binary: 0.3676 | Regression_cos_phi: 0.2302 | Regression_energy: 0.1897 | Regression_eta: 0.1855 | Regression_pt: 0.1989 | Regression_sin_phi: 0.2710 | LR: 9.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4725s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank3 [2026-10-28 16:00:38,672] INFO : Running validation on rank3 +rank3 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:01:32,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:01:33,505] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 16:02:00,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:02:01,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:02:02,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:02:02,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:02:12,968] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:02:14,306] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 16:02:30,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:02:31,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:02:32,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:02:32,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:02:42,975] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:02:43,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:02:44,279] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 16:02:59,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:03:00,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:03:01,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:03:01,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:03:11,901] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank3 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1329 | Classification_binary: 0.2739 | Regression_cos_phi: 0.1835 | Regression_energy: 0.1578 | Regression_eta: 0.1565 | Regression_pt: 0.1664 | Regression_sin_phi: 0.2242 | LR: 9.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5607s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank3 [2026-10-28 16:11:41,309] INFO : Step 7200/50000 rank3 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3368 | Regression_cos_phi: 0.1958 | Regression_energy: 0.1844 | Regression_eta: 0.1856 | Regression_pt: 0.1921 | Regression_sin_phi: 0.1888 | LR: 9.55e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9561s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank3 [2026-10-28 16:15:45,431] INFO : Step 7300/50000 rank3 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1409 | Classification_binary: 0.3600 | Regression_cos_phi: 0.2378 | Regression_energy: 0.1872 | Regression_eta: 0.1969 | Regression_pt: 0.1963 | Regression_sin_phi: 0.2004 | LR: 9.53e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8992s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank3 [2026-10-28 16:19:47,790] INFO : Step 7400/50000 rank3 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2890 | Regression_cos_phi: 0.2530 | Regression_energy: 0.1815 | Regression_eta: 0.1447 | Regression_pt: 0.1901 | Regression_sin_phi: 0.1366 | LR: 9.52e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6017s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank3 [2026-10-28 16:23:56,336] INFO : Step 7500/50000 rank3 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1206 | Classification_binary: 0.2837 | Regression_cos_phi: 0.2174 | Regression_energy: 0.1800 | Regression_eta: 0.1371 | Regression_pt: 0.1891 | Regression_sin_phi: 0.1669 | LR: 9.51e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7918s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank3 [2026-10-28 16:27:58,400] INFO : Step 7600/50000 rank3 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1318 | Classification_binary: 0.3734 | Regression_cos_phi: 0.1647 | Regression_energy: 0.1531 | Regression_eta: 0.1642 | Regression_pt: 0.1593 | Regression_sin_phi: 0.2329 | LR: 9.50e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7013s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank3 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank3 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1164 | Classification_binary: 0.3403 | Regression_cos_phi: 0.1487 | Regression_energy: 0.1665 | Regression_eta: 0.1473 | Regression_pt: 0.1750 | Regression_sin_phi: 0.1694 | LR: 9.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9796s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank3 [2026-10-28 16:35:55,393] INFO : Step 7800/50000 rank3 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.2585 | Regression_cos_phi: 0.1499 | Regression_energy: 0.1572 | Regression_eta: 0.1268 | Regression_pt: 0.1644 | Regression_sin_phi: 0.1570 | LR: 9.47e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2109s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank3 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank3 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1002 | Classification_binary: 0.3111 | Regression_cos_phi: 0.1806 | Regression_energy: 0.1464 | Regression_eta: 0.1261 | Regression_pt: 0.1499 | Regression_sin_phi: 0.1690 | LR: 9.46e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.3423s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank3 [2026-10-28 16:44:12,790] INFO : Step 8000/50000 rank3 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.3403 | Regression_cos_phi: 0.1809 | Regression_energy: 0.1524 | Regression_eta: 0.1467 | Regression_pt: 0.1580 | Regression_sin_phi: 0.1863 | LR: 9.44e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2450s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank3 [2026-10-28 16:44:13,936] INFO : Running validation on rank3 +rank3 [2026-10-28 16:44:13,936] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:44:13,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:45:04,006] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:45:04,567] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 16:45:30,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:45:31,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:45:32,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:45:34,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:45:44,073] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:45:45,682] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 16:46:01,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:46:02,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:46:03,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:46:04,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:46:14,537] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:46:15,125] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 16:46:30,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:46:31,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:46:32,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:46:33,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:46:43,013] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank3 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.2769 | Regression_cos_phi: 0.1603 | Regression_energy: 0.1546 | Regression_eta: 0.1159 | Regression_pt: 0.1580 | Regression_sin_phi: 0.1718 | LR: 9.43e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7757s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank3 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank3 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2456 | Regression_cos_phi: 0.1273 | Regression_energy: 0.1509 | Regression_eta: 0.1305 | Regression_pt: 0.1577 | Regression_sin_phi: 0.1521 | LR: 9.42e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1552s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank3 [2026-10-28 16:59:31,718] INFO : Step 8300/50000 rank3 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1422 | Classification_binary: 0.4057 | Regression_cos_phi: 0.2437 | Regression_energy: 0.1647 | Regression_eta: 0.1919 | Regression_pt: 0.1734 | Regression_sin_phi: 0.2175 | LR: 9.40e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.9199s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank3 [2026-10-28 17:03:26,315] INFO : Step 8400/50000 rank3 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 0.2955 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1681 | Regression_eta: 0.1533 | Regression_pt: 0.1757 | Regression_sin_phi: 0.1673 | LR: 9.39e-04 | DataLoad Time: 0.0221s | Model Forward Time: 6.0456s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank3 [2026-10-28 17:07:18,316] INFO : Step 8500/50000 rank3 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1279 | Classification_binary: 0.2703 | Regression_cos_phi: 0.1307 | Regression_energy: 0.1632 | Regression_eta: 0.1344 | Regression_pt: 0.1695 | Regression_sin_phi: 0.1734 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4973s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank3 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank3 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.2906 | Regression_cos_phi: 0.1138 | Regression_energy: 0.1594 | Regression_eta: 0.1008 | Regression_pt: 0.1677 | Regression_sin_phi: 0.1063 | LR: 9.36e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1686s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank3 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank3 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1137 | Classification_binary: 0.3261 | Regression_cos_phi: 0.1536 | Regression_energy: 0.1589 | Regression_eta: 0.1442 | Regression_pt: 0.1643 | Regression_sin_phi: 0.1779 | LR: 9.34e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2146s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank3 [2026-10-28 17:19:40,248] INFO : Step 8800/50000 rank3 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2200 | Regression_cos_phi: 0.1478 | Regression_energy: 0.1371 | Regression_eta: 0.0866 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1360 | LR: 9.33e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.4426s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank3 [2026-10-28 17:23:38,392] INFO : Step 8900/50000 rank3 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0908 | Classification_binary: 0.2976 | Regression_cos_phi: 0.1350 | Regression_energy: 0.1685 | Regression_eta: 0.1070 | Regression_pt: 0.1746 | Regression_sin_phi: 0.1323 | LR: 9.31e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6548s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank3 [2026-10-28 17:27:33,248] INFO : Step 9000/50000 rank3 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.3400 | Regression_cos_phi: 0.1431 | Regression_energy: 0.1463 | Regression_eta: 0.1488 | Regression_pt: 0.1495 | Regression_sin_phi: 0.1396 | LR: 9.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8288s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank3 [2026-10-28 17:27:34,333] INFO : Running validation on rank3 +rank3 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 17:28:24,727] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:28:25,552] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 17:28:52,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 17:28:53,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 17:28:54,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 17:28:54,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 17:29:04,971] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:29:06,075] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 17:29:22,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 17:29:23,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 17:29:24,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 17:29:25,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 17:29:35,458] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:29:37,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:29:37,692] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 17:29:53,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 17:29:54,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 17:29:55,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 17:29:55,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 17:30:05,721] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 17:34:23,899] INFO : Step 9100/50000 rank3 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2826 | Regression_cos_phi: 0.1228 | Regression_energy: 0.1410 | Regression_eta: 0.1235 | Regression_pt: 0.1453 | Regression_sin_phi: 0.1562 | LR: 9.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1062s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank3 [2026-10-28 17:38:24,814] INFO : Step 9200/50000 rank3 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1132 | Classification_binary: 0.2958 | Regression_cos_phi: 0.1501 | Regression_energy: 0.1575 | Regression_eta: 0.1488 | Regression_pt: 0.1682 | Regression_sin_phi: 0.1992 | LR: 9.27e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3256s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank3 [2026-10-28 17:42:44,641] INFO : Step 9300/50000 rank3 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.3879 | Regression_cos_phi: 0.2031 | Regression_energy: 0.1627 | Regression_eta: 0.1488 | Regression_pt: 0.1737 | Regression_sin_phi: 0.1485 | LR: 9.25e-04 | DataLoad Time: 0.0244s | Model Forward Time: 4.8249s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank3 [2026-10-28 17:46:50,463] INFO : Step 9400/50000 rank3 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1261 | Classification_binary: 0.2868 | Regression_cos_phi: 0.1367 | Regression_energy: 0.1468 | Regression_eta: 0.1230 | Regression_pt: 0.1517 | Regression_sin_phi: 0.1991 | LR: 9.24e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4139s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank3 [2026-10-28 17:51:11,695] INFO : Step 9500/50000 rank3 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2475 | Regression_cos_phi: 0.1099 | Regression_energy: 0.1287 | Regression_eta: 0.0986 | Regression_pt: 0.1334 | Regression_sin_phi: 0.1207 | LR: 9.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.2536s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank3 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank3 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1185 | Classification_binary: 0.2772 | Regression_cos_phi: 0.1811 | Regression_energy: 0.1493 | Regression_eta: 0.1235 | Regression_pt: 0.1559 | Regression_sin_phi: 0.1698 | LR: 9.21e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.8397s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank3 [2026-10-28 17:59:31,910] INFO : Step 9700/50000 rank3 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1027 | Classification_binary: 0.3541 | Regression_cos_phi: 0.1459 | Regression_energy: 0.1246 | Regression_eta: 0.1273 | Regression_pt: 0.1316 | Regression_sin_phi: 0.1292 | LR: 9.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9931s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank3 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank3 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.2473 | Regression_cos_phi: 0.1411 | Regression_energy: 0.1174 | Regression_eta: 0.0977 | Regression_pt: 0.1238 | Regression_sin_phi: 0.1258 | LR: 9.17e-04 | DataLoad Time: 0.0222s | Model Forward Time: 3.6867s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank3 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank3 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0978 | Classification_binary: 0.2858 | Regression_cos_phi: 0.1653 | Regression_energy: 0.1337 | Regression_eta: 0.1556 | Regression_pt: 0.1415 | Regression_sin_phi: 0.1326 | LR: 9.16e-04 | DataLoad Time: 0.0128s | Model Forward Time: 4.1799s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank3 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank3 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.3532 | Regression_cos_phi: 0.1777 | Regression_energy: 0.1402 | Regression_eta: 0.1549 | Regression_pt: 0.1457 | Regression_sin_phi: 0.1613 | LR: 9.14e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1265s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank3 [2026-10-28 18:11:36,666] INFO : Running validation on rank3 +rank3 [2026-10-28 18:11:36,666] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:11:36,666] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:12:31,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:12:31,988] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 18:12:58,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:12:59,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:13:00,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:13:01,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:13:11,214] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 18:13:12,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:13:12,693] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 18:13:32,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:13:36,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:13:37,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:13:38,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:13:48,511] INFO : Time taken to make predictions on device 3 is: 0.37 min +rank3 [2026-10-28 18:13:48,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:13:49,086] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 18:14:04,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:14:05,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:14:06,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:14:06,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:14:16,854] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 18:18:32,574] INFO : Step 10100/50000 rank3 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.2798 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1491 | Regression_eta: 0.0841 | Regression_pt: 0.1545 | Regression_sin_phi: 0.1086 | LR: 9.12e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.2349s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank3 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank3 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.2597 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1579 | Regression_eta: 0.1070 | Regression_pt: 0.1603 | Regression_sin_phi: 0.1269 | LR: 9.11e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5705s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank3 [2026-10-28 18:26:52,298] INFO : Step 10300/50000 rank3 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.3599 | Regression_cos_phi: 0.1348 | Regression_energy: 0.1351 | Regression_eta: 0.1156 | Regression_pt: 0.1416 | Regression_sin_phi: 0.1494 | LR: 9.09e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.3471s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank3 [2026-10-28 18:31:09,157] INFO : Step 10400/50000 rank3 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1028 | Classification_binary: 0.3388 | Regression_cos_phi: 0.1255 | Regression_energy: 0.1168 | Regression_eta: 0.1035 | Regression_pt: 0.1232 | Regression_sin_phi: 0.1246 | LR: 9.07e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9062s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank3 [2026-10-28 18:35:25,945] INFO : Step 10500/50000 rank3 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2285 | Regression_cos_phi: 0.1639 | Regression_energy: 0.1197 | Regression_eta: 0.1062 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1336 | LR: 9.06e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.8285s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank3 [2026-10-28 18:39:26,894] INFO : Step 10600/50000 rank3 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.2651 | Regression_cos_phi: 0.1135 | Regression_energy: 0.1169 | Regression_eta: 0.1039 | Regression_pt: 0.1222 | Regression_sin_phi: 0.1064 | LR: 9.04e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.2176s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank3 [2026-10-28 18:43:27,488] INFO : Step 10700/50000 rank3 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0900 | Classification_binary: 0.3057 | Regression_cos_phi: 0.1329 | Regression_energy: 0.1225 | Regression_eta: 0.0995 | Regression_pt: 0.1265 | Regression_sin_phi: 0.1219 | LR: 9.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2752s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank3 [2026-10-28 18:47:34,478] INFO : Step 10800/50000 rank3 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2450 | Regression_cos_phi: 0.1141 | Regression_energy: 0.1438 | Regression_eta: 0.0888 | Regression_pt: 0.1498 | Regression_sin_phi: 0.1063 | LR: 9.00e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.3718s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank3 [2026-10-28 18:51:51,744] INFO : Step 10900/50000 rank3 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0893 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1182 | Regression_energy: 0.1163 | Regression_eta: 0.1059 | Regression_pt: 0.1204 | Regression_sin_phi: 0.1170 | LR: 8.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6020s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank3 [2026-10-28 18:55:59,248] INFO : Step 11000/50000 rank3 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1436 | Regression_eta: 0.0951 | Regression_pt: 0.1489 | Regression_sin_phi: 0.1007 | LR: 8.97e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.8660s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank3 [2026-10-28 18:56:00,216] INFO : Running validation on rank3 +rank3 [2026-10-28 18:56:00,216] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:56:00,216] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:56:54,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:56:55,375] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 18:57:21,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:57:22,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:57:23,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:57:24,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:57:34,810] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 18:57:35,271] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:57:35,836] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 18:57:51,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:57:52,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:57:54,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:57:55,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:58:05,138] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:58:06,770] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 18:58:22,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:58:23,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:58:23,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:58:24,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:58:34,557] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 19:03:04,100] INFO : Step 11100/50000 rank3 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2625 | Regression_cos_phi: 0.0801 | Regression_energy: 0.1183 | Regression_eta: 0.0772 | Regression_pt: 0.1220 | Regression_sin_phi: 0.1006 | LR: 8.95e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.4752s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank3 [2026-10-28 19:07:10,330] INFO : Step 11200/50000 rank3 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0987 | Regression_energy: 0.1405 | Regression_eta: 0.1085 | Regression_pt: 0.1495 | Regression_sin_phi: 0.1069 | LR: 8.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8098s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank3 [2026-10-28 19:11:04,857] INFO : Step 11300/50000 rank3 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2633 | Regression_cos_phi: 0.1015 | Regression_energy: 0.1379 | Regression_eta: 0.0778 | Regression_pt: 0.1428 | Regression_sin_phi: 0.1160 | LR: 8.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2489s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank3 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank3 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2825 | Regression_cos_phi: 0.0802 | Regression_energy: 0.1250 | Regression_eta: 0.1060 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0969 | LR: 8.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0027s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank3 [2026-10-28 19:18:58,943] INFO : Step 11500/50000 rank3 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2488 | Regression_cos_phi: 0.1200 | Regression_energy: 0.1083 | Regression_eta: 0.0867 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0987 | LR: 8.88e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.4533s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank3 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank3 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0997 | Classification_binary: 0.2932 | Regression_cos_phi: 0.1197 | Regression_energy: 0.1334 | Regression_eta: 0.1066 | Regression_pt: 0.1376 | Regression_sin_phi: 0.1025 | LR: 8.86e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8311s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank3 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank3 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2977 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1139 | Regression_eta: 0.0997 | Regression_pt: 0.1169 | Regression_sin_phi: 0.1020 | LR: 8.84e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1778s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank3 [2026-10-28 19:30:54,022] INFO : Step 11800/50000 rank3 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2738 | Regression_cos_phi: 0.1079 | Regression_energy: 0.1182 | Regression_eta: 0.0981 | Regression_pt: 0.1236 | Regression_sin_phi: 0.1268 | LR: 8.82e-04 | DataLoad Time: 0.0238s | Model Forward Time: 6.0648s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank3 [2026-10-28 19:34:55,960] INFO : Step 11900/50000 rank3 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0938 | Classification_binary: 0.2381 | Regression_cos_phi: 0.1231 | Regression_energy: 0.1224 | Regression_eta: 0.0977 | Regression_pt: 0.1308 | Regression_sin_phi: 0.1271 | LR: 8.80e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9524s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank3 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank3 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.3127 | Regression_cos_phi: 0.0799 | Regression_energy: 0.1086 | Regression_eta: 0.0775 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0957 | LR: 8.78e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3560s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank3 [2026-10-28 19:39:04,976] INFO : Running validation on rank3 +rank3 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 19:39:55,047] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:39:55,583] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 19:40:21,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 19:40:22,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 19:40:23,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 19:40:24,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 19:40:34,382] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 19:40:34,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:40:35,408] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 19:40:51,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 19:40:52,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 19:40:53,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 19:40:54,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 19:41:04,053] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 19:41:05,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:41:05,608] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 19:41:21,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 19:41:22,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 19:41:23,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 19:41:23,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 19:41:33,773] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 19:45:57,645] INFO : Step 12100/50000 rank3 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2622 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1116 | Regression_eta: 0.0927 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1226 | LR: 8.76e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8594s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank3 [2026-10-28 19:50:02,389] INFO : Step 12200/50000 rank3 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0682 | Regression_energy: 0.1287 | Regression_eta: 0.0654 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0625 | LR: 8.74e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.9605s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank3 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank3 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1089 | Classification_binary: 0.2732 | Regression_cos_phi: 0.0981 | Regression_energy: 0.1002 | Regression_eta: 0.0929 | Regression_pt: 0.1024 | Regression_sin_phi: 0.1065 | LR: 8.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4597s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank3 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank3 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.3186 | Regression_cos_phi: 0.1180 | Regression_energy: 0.1052 | Regression_eta: 0.1034 | Regression_pt: 0.1088 | Regression_sin_phi: 0.1433 | LR: 8.70e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9486s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank3 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank3 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0802 | Regression_energy: 0.1117 | Regression_eta: 0.0735 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0810 | LR: 8.68e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.6934s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank3 [2026-10-28 20:06:38,788] INFO : Step 12600/50000 rank3 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2530 | Regression_cos_phi: 0.0819 | Regression_energy: 0.1040 | Regression_eta: 0.0666 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0960 | LR: 8.66e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9515s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank3 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank3 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1205 | Regression_energy: 0.0909 | Regression_eta: 0.0863 | Regression_pt: 0.0955 | Regression_sin_phi: 0.1049 | LR: 8.64e-04 | DataLoad Time: 0.0089s | Model Forward Time: 5.2459s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank3 [2026-10-28 20:14:52,957] INFO : Step 12800/50000 rank3 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2260 | Regression_cos_phi: 0.0804 | Regression_energy: 0.1269 | Regression_eta: 0.0682 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0746 | LR: 8.62e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8532s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank3 [2026-10-28 20:19:01,948] INFO : Step 12900/50000 rank3 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0809 | Regression_energy: 0.1024 | Regression_eta: 0.0547 | Regression_pt: 0.1035 | Regression_sin_phi: 0.0647 | LR: 8.60e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.5186s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank3 [2026-10-28 20:23:25,486] INFO : Step 13000/50000 rank3 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2794 | Regression_cos_phi: 0.0767 | Regression_energy: 0.1205 | Regression_eta: 0.0648 | Regression_pt: 0.1255 | Regression_sin_phi: 0.1048 | LR: 8.58e-04 | DataLoad Time: 0.0159s | Model Forward Time: 10.3192s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank3 [2026-10-28 20:23:26,430] INFO : Running validation on rank3 +rank3 [2026-10-28 20:23:26,430] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 20:23:26,430] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 20:24:18,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 20:24:45,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 20:24:46,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 20:24:47,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 20:24:48,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 20:24:58,842] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 20:25:04,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:25:04,757] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 20:25:20,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 20:25:21,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 20:25:22,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 20:25:23,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 20:25:33,233] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:25:34,723] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 20:25:51,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 20:25:51,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 20:25:52,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 20:25:53,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 20:26:03,185] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 20:30:28,057] INFO : Step 13100/50000 rank3 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2706 | Regression_cos_phi: 0.1395 | Regression_energy: 0.1172 | Regression_eta: 0.0917 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1074 | LR: 8.56e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.0199s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank3 [2026-10-28 20:34:38,790] INFO : Step 13200/50000 rank3 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2339 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0904 | Regression_eta: 0.0945 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0984 | LR: 8.54e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.0432s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank3 [2026-10-28 20:38:50,942] INFO : Step 13300/50000 rank3 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.2868 | Regression_cos_phi: 0.1317 | Regression_energy: 0.1027 | Regression_eta: 0.1104 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0888 | LR: 8.52e-04 | DataLoad Time: 0.0150s | Model Forward Time: 6.4749s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank3 [2026-10-28 20:43:04,348] INFO : Step 13400/50000 rank3 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0786 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1254 | Regression_energy: 0.1006 | Regression_eta: 0.1055 | Regression_pt: 0.1058 | Regression_sin_phi: 0.1324 | LR: 8.50e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2845s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank3 [2026-10-28 20:47:13,648] INFO : Step 13500/50000 rank3 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2478 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1020 | Regression_eta: 0.0802 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0862 | LR: 8.48e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1209s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank3 [2026-10-28 20:51:26,252] INFO : Step 13600/50000 rank3 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2789 | Regression_cos_phi: 0.1187 | Regression_energy: 0.0823 | Regression_eta: 0.0917 | Regression_pt: 0.0868 | Regression_sin_phi: 0.1012 | LR: 8.45e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.8106s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank3 [2026-10-28 20:55:41,371] INFO : Step 13700/50000 rank3 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.3138 | Regression_cos_phi: 0.1116 | Regression_energy: 0.1041 | Regression_eta: 0.0901 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1250 | LR: 8.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.7989s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank3 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank3 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2402 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0970 | Regression_eta: 0.0676 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0825 | LR: 8.41e-04 | DataLoad Time: 0.0214s | Model Forward Time: 6.1150s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank3 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank3 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0731 | Regression_eta: 0.0479 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0620 | LR: 8.39e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1596s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank3 [2026-10-28 21:08:24,062] INFO : Step 14000/50000 rank3 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2727 | Regression_cos_phi: 0.1138 | Regression_energy: 0.1080 | Regression_eta: 0.0682 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1060 | LR: 8.37e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.9198s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank3 [2026-10-28 21:08:24,997] INFO : Running validation on rank3 +rank3 [2026-10-28 21:08:24,997] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:09:16,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 21:09:43,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:09:43,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:09:44,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:09:45,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:09:55,841] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 21:09:56,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:09:56,996] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 21:10:13,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:10:14,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:10:15,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:10:16,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:10:26,178] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:10:26,836] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 21:10:43,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:10:43,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:10:45,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:10:46,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:10:56,260] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank3 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.3143 | Regression_cos_phi: 0.1312 | Regression_energy: 0.0967 | Regression_eta: 0.0774 | Regression_pt: 0.1025 | Regression_sin_phi: 0.1145 | LR: 8.35e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank3 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank3 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.2026 | Regression_cos_phi: 0.0775 | Regression_energy: 0.1181 | Regression_eta: 0.0753 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0755 | LR: 8.32e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4856s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank3 [2026-10-28 21:24:03,128] INFO : Step 14300/50000 rank3 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2463 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0877 | Regression_eta: 0.0871 | Regression_pt: 0.0925 | Regression_sin_phi: 0.1593 | LR: 8.30e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0218s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank3 [2026-10-28 21:28:09,727] INFO : Step 14400/50000 rank3 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2674 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0940 | Regression_eta: 0.0682 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0843 | LR: 8.28e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0269s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank3 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank3 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2135 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0973 | Regression_eta: 0.0542 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0506 | LR: 8.26e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.3483s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank3 [2026-10-28 21:36:11,051] INFO : Step 14600/50000 rank3 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0897 | Regression_eta: 0.0731 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0992 | LR: 8.24e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.9784s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank3 [2026-10-28 21:40:16,607] INFO : Step 14700/50000 rank3 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.3007 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1006 | Regression_eta: 0.0860 | Regression_pt: 0.1050 | Regression_sin_phi: 0.1155 | LR: 8.21e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.1355s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank3 [2026-10-28 21:44:33,910] INFO : Step 14800/50000 rank3 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0974 | Regression_eta: 0.0599 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0885 | LR: 8.19e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3495s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank3 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank3 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0867 | Regression_eta: 0.0617 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0857 | LR: 8.17e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.3230s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank3 [2026-10-28 21:52:53,292] INFO : Step 15000/50000 rank3 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2557 | Regression_cos_phi: 0.1114 | Regression_energy: 0.0946 | Regression_eta: 0.0760 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0875 | LR: 8.15e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7244s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank3 [2026-10-28 21:52:54,209] INFO : Running validation on rank3 +rank3 [2026-10-28 21:52:54,209] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:53:44,051] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:53:44,570] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 21:54:10,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:54:11,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:54:12,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:54:13,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:54:23,725] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:54:24,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:54:24,599] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 21:54:40,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:54:41,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:54:42,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:54:43,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:54:53,445] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:54:54,913] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 21:55:10,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:55:11,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:55:11,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:55:12,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:55:22,745] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 21:59:40,817] INFO : Step 15100/50000 rank3 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0884 | Classification_binary: 0.2468 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0780 | Regression_eta: 0.0976 | Regression_pt: 0.0849 | Regression_sin_phi: 0.1341 | LR: 8.12e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.6885s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank3 [2026-10-28 22:03:40,545] INFO : Step 15200/50000 rank3 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0827 | Regression_eta: 0.1288 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0874 | LR: 8.10e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9701s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank3 [2026-10-28 22:07:35,555] INFO : Step 15300/50000 rank3 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0741 | Regression_eta: 0.0626 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0728 | LR: 8.08e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7977s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank3 [2026-10-28 22:11:47,342] INFO : Step 15400/50000 rank3 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2901 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0979 | Regression_eta: 0.0896 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0798 | LR: 8.05e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1750s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank3 [2026-10-28 22:16:06,324] INFO : Step 15500/50000 rank3 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2283 | Regression_cos_phi: 0.0952 | Regression_energy: 0.0960 | Regression_eta: 0.0598 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0542 | LR: 8.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9204s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank3 [2026-10-28 22:20:18,326] INFO : Step 15600/50000 rank3 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0864 | Regression_eta: 0.0694 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0661 | LR: 8.01e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.4204s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank3 [2026-10-28 22:24:30,251] INFO : Step 15700/50000 rank3 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2709 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0928 | Regression_eta: 0.0778 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0978 | LR: 7.98e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.9287s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank3 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank3 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2482 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0977 | Regression_eta: 0.0804 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0883 | LR: 7.96e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3024s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank3 [2026-10-28 22:32:57,711] INFO : Step 15900/50000 rank3 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0826 | Regression_eta: 0.0662 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0850 | LR: 7.94e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.3907s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank3 [2026-10-28 22:37:10,742] INFO : Step 16000/50000 rank3 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2220 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0974 | Regression_eta: 0.0615 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0715 | LR: 7.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0613s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank3 [2026-10-28 22:37:11,680] INFO : Running validation on rank3 +rank3 [2026-10-28 22:37:11,680] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 22:37:11,680] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 22:38:03,506] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:38:04,032] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 22:38:30,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 22:38:31,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 22:38:32,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 22:38:33,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 22:38:43,158] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:38:43,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:38:44,322] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 22:39:00,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 22:39:01,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 22:39:02,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 22:39:03,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 22:39:13,855] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:39:15,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:39:16,105] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 22:39:31,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 22:39:32,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 22:39:33,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 22:39:33,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 22:39:43,904] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 22:44:03,865] INFO : Step 16100/50000 rank3 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2775 | Regression_cos_phi: 0.1319 | Regression_energy: 0.0997 | Regression_eta: 0.0891 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1771 | LR: 7.89e-04 | DataLoad Time: 0.0091s | Model Forward Time: 4.7902s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank3 [2026-10-28 22:48:09,286] INFO : Step 16200/50000 rank3 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2154 | Regression_cos_phi: 0.1168 | Regression_energy: 0.0925 | Regression_eta: 0.0733 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0694 | LR: 7.86e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.6654s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank3 [2026-10-28 22:52:16,253] INFO : Step 16300/50000 rank3 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2384 | Regression_cos_phi: 0.0757 | Regression_energy: 0.1017 | Regression_eta: 0.0546 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0969 | LR: 7.84e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1375s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank3 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank3 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2920 | Regression_cos_phi: 0.1518 | Regression_energy: 0.0840 | Regression_eta: 0.1185 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0956 | LR: 7.81e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4171s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank3 [2026-10-28 23:00:23,723] INFO : Step 16500/50000 rank3 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2102 | Regression_cos_phi: 0.1216 | Regression_energy: 0.0947 | Regression_eta: 0.0705 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0809 | LR: 7.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9994s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank3 [2026-10-28 23:04:26,317] INFO : Step 16600/50000 rank3 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0944 | Regression_eta: 0.0703 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0697 | LR: 7.77e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.3627s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank3 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank3 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1219 | Regression_energy: 0.0950 | Regression_eta: 0.1088 | Regression_pt: 0.0982 | Regression_sin_phi: 0.1229 | LR: 7.74e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0573s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank3 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank3 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0869 | Regression_eta: 0.0837 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0796 | LR: 7.72e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.0649s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank3 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank3 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0891 | Regression_eta: 0.0556 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0547 | LR: 7.69e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.7652s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank3 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank3 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0900 | Regression_eta: 0.0473 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0515 | LR: 7.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9352s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank3 [2026-10-28 23:20:37,935] INFO : Running validation on rank3 +rank3 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 23:21:28,541] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:21:29,099] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 23:21:55,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 23:21:57,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 23:21:59,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 23:22:01,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 23:22:11,050] INFO : Time taken to make predictions on device 3 is: 0.30 min +rank3 [2026-10-28 23:22:12,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:22:12,959] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 23:22:28,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 23:22:29,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 23:22:30,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 23:22:31,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 23:22:41,729] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 23:22:42,340] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:22:42,878] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 23:22:58,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 23:22:59,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 23:23:00,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 23:23:00,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 23:23:10,937] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 23:27:38,314] INFO : Step 17100/50000 rank3 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2644 | Regression_cos_phi: 0.1177 | Regression_energy: 0.0872 | Regression_eta: 0.0931 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0856 | LR: 7.64e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1663s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank3 [2026-10-28 23:31:38,671] INFO : Step 17200/50000 rank3 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0676 | Regression_eta: 0.0821 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0893 | LR: 7.62e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1942s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank3 [2026-10-28 23:35:49,544] INFO : Step 17300/50000 rank3 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0731 | Regression_eta: 0.0728 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0724 | LR: 7.59e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.5110s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank3 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank3 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2873 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0708 | Regression_eta: 0.0793 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0834 | LR: 7.57e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.3513s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank3 [2026-10-28 23:43:53,977] INFO : Step 17500/50000 rank3 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2518 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0936 | Regression_eta: 0.0595 | Regression_pt: 0.0962 | Regression_sin_phi: 0.1053 | LR: 7.54e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9648s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank3 [2026-10-28 23:48:08,874] INFO : Step 17600/50000 rank3 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0842 | Regression_eta: 0.0688 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0830 | LR: 7.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5569s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank3 [2026-10-28 23:52:17,699] INFO : Step 17700/50000 rank3 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0878 | Regression_eta: 0.0686 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0890 | LR: 7.49e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.8659s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank3 [2026-10-28 23:56:40,920] INFO : Step 17800/50000 rank3 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2598 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0876 | Regression_eta: 0.0830 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0957 | LR: 7.47e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1268s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank3 [2026-10-29 00:00:47,129] INFO : Step 17900/50000 rank3 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1786 | Regression_cos_phi: 0.1048 | Regression_energy: 0.0604 | Regression_eta: 0.0872 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0758 | LR: 7.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2712s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank3 [2026-10-29 00:04:53,688] INFO : Step 18000/50000 rank3 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.2118 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0815 | Regression_eta: 0.0911 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0697 | LR: 7.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 13.5870s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank3 [2026-10-29 00:04:54,630] INFO : Running validation on rank3 +rank3 [2026-10-29 00:04:54,630] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 00:04:54,630] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:05:43,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:05:43,819] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 00:06:10,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:06:11,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:06:13,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:06:14,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:06:24,357] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 00:06:24,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:06:25,296] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 00:06:41,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:06:42,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:06:43,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:06:44,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:06:54,294] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 00:07:11,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:07:12,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:07:12,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:07:13,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:07:23,686] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 00:11:38,771] INFO : Step 18100/50000 rank3 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2401 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0824 | Regression_eta: 0.0911 | Regression_pt: 0.0831 | Regression_sin_phi: 0.1104 | LR: 7.39e-04 | DataLoad Time: 0.0248s | Model Forward Time: 4.2921s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank3 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank3 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2225 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0785 | Regression_eta: 0.0491 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0536 | LR: 7.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1147s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank3 [2026-10-29 00:19:54,086] INFO : Step 18300/50000 rank3 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0656 | Regression_eta: 0.0571 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0935 | LR: 7.34e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8053s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank3 [2026-10-29 00:24:07,389] INFO : Step 18400/50000 rank3 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2092 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0786 | Regression_eta: 0.0620 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0830 | LR: 7.31e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4800s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank3 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank3 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0857 | Regression_eta: 0.0662 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0955 | LR: 7.29e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5455s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank3 [2026-10-29 00:31:24,472] INFO : Step 18600/50000 rank3 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0749 | Regression_eta: 0.0637 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0600 | LR: 7.26e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1900s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank3 [2026-10-29 00:35:06,906] INFO : Step 18700/50000 rank3 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0815 | Regression_eta: 0.0413 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0716 | LR: 7.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.9497s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank3 [2026-10-29 00:38:49,580] INFO : Step 18800/50000 rank3 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2348 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0714 | Regression_eta: 0.0601 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0733 | LR: 7.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4356s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank3 [2026-10-29 00:42:28,237] INFO : Step 18900/50000 rank3 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0633 | Regression_eta: 0.0615 | Regression_pt: 0.0691 | Regression_sin_phi: 0.1021 | LR: 7.18e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0104s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank3 [2026-10-29 00:46:10,110] INFO : Step 19000/50000 rank3 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2283 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0790 | Regression_eta: 0.0715 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0869 | LR: 7.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1537s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank3 [2026-10-29 00:46:11,054] INFO : Running validation on rank3 +rank3 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:46:58,593] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:46:59,173] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 00:47:25,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:47:26,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:47:27,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:47:28,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:47:38,050] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:47:38,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:47:38,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 00:47:54,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:47:55,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:47:56,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:47:57,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:48:07,549] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:48:09,173] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 00:48:25,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:48:25,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:48:26,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:48:27,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:48:37,165] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank3 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0783 | Regression_eta: 0.0877 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0836 | LR: 7.13e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5906s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank3 [2026-10-29 00:56:31,490] INFO : Step 19200/50000 rank3 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0714 | Regression_eta: 0.0652 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0820 | LR: 7.10e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7728s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank3 [2026-10-29 01:00:15,644] INFO : Step 19300/50000 rank3 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1952 | Regression_cos_phi: 0.1135 | Regression_energy: 0.0877 | Regression_eta: 0.0999 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0883 | LR: 7.08e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6468s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank3 [2026-10-29 01:03:57,848] INFO : Step 19400/50000 rank3 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0810 | Regression_eta: 0.0756 | Regression_pt: 0.0858 | Regression_sin_phi: 0.1102 | LR: 7.05e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1733s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank3 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank3 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2208 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0837 | Regression_eta: 0.0776 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0766 | LR: 7.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0514s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank3 [2026-10-29 01:11:25,841] INFO : Step 19600/50000 rank3 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0869 | Regression_eta: 0.0502 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0487 | LR: 7.00e-04 | DataLoad Time: 0.0222s | Model Forward Time: 3.9372s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank3 [2026-10-29 01:15:28,502] INFO : Step 19700/50000 rank3 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.2024 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0721 | Regression_eta: 0.0508 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0815 | LR: 6.97e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.3541s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank3 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank3 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2442 | Regression_cos_phi: 0.0578 | Regression_energy: 0.1023 | Regression_eta: 0.0731 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0875 | LR: 6.94e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3398s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank3 [2026-10-29 01:23:26,315] INFO : Step 19900/50000 rank3 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0817 | Regression_eta: 0.0812 | Regression_pt: 0.0835 | Regression_sin_phi: 0.1069 | LR: 6.92e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7487s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank3 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank3 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0956 | Regression_energy: 0.1041 | Regression_eta: 0.0841 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0944 | LR: 6.89e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8750s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank3 [2026-10-29 01:27:16,848] INFO : Running validation on rank3 +rank3 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 01:28:06,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:28:07,178] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 01:28:33,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 01:28:34,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 01:28:35,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 01:28:37,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 01:28:47,056] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 01:28:48,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:28:48,587] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 01:29:04,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 01:29:05,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 01:29:06,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 01:29:07,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 01:29:17,626] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 01:29:18,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:29:18,550] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 01:29:34,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 01:29:34,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 01:29:35,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 01:29:36,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 01:29:46,366] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 01:33:54,475] INFO : Step 20100/50000 rank3 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.2537 | Regression_cos_phi: 0.1204 | Regression_energy: 0.1005 | Regression_eta: 0.0862 | Regression_pt: 0.1060 | Regression_sin_phi: 0.1016 | LR: 6.86e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4722s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank3 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank3 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1976 | Regression_cos_phi: 0.1006 | Regression_energy: 0.0817 | Regression_eta: 0.0716 | Regression_pt: 0.0847 | Regression_sin_phi: 0.1132 | LR: 6.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9216s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank3 [2026-10-29 01:41:55,631] INFO : Step 20300/50000 rank3 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0769 | Regression_eta: 0.0651 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0946 | LR: 6.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.6958s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank3 [2026-10-29 01:45:46,290] INFO : Step 20400/50000 rank3 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2393 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0760 | Regression_eta: 0.0762 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0751 | LR: 6.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1300s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank3 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank3 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2446 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0759 | Regression_eta: 0.0790 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0506 | LR: 6.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3413s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank3 [2026-10-29 01:53:32,129] INFO : Step 20600/50000 rank3 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2001 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0860 | Regression_eta: 0.0627 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0804 | LR: 6.73e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.0699s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank3 [2026-10-29 01:57:29,940] INFO : Step 20700/50000 rank3 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.1720 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0867 | Regression_eta: 0.0913 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0827 | LR: 6.70e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.6597s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank3 [2026-10-29 02:01:35,425] INFO : Step 20800/50000 rank3 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2192 | Regression_cos_phi: 0.0800 | Regression_energy: 0.1036 | Regression_eta: 0.0760 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0750 | LR: 6.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8011s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank3 [2026-10-29 02:05:49,595] INFO : Step 20900/50000 rank3 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0738 | Regression_eta: 0.0586 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0645 | LR: 6.65e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.1308s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank3 [2026-10-29 02:09:53,644] INFO : Step 21000/50000 rank3 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2265 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0950 | Regression_eta: 0.0774 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0963 | LR: 6.62e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8420s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank3 [2026-10-29 02:09:54,610] INFO : Running validation on rank3 +rank3 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:10:45,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:10:45,900] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 02:11:12,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:11:13,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:11:14,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:11:15,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:11:25,364] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:11:26,308] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 02:11:43,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:11:44,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:11:45,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:11:47,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:11:57,170] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 02:11:57,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:11:57,988] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 02:12:13,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:12:14,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:12:15,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:12:15,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:12:25,730] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 02:16:43,090] INFO : Step 21100/50000 rank3 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0829 | Regression_eta: 0.0867 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0859 | LR: 6.59e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank3 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank3 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank3 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0701 | Regression_eta: 0.0598 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0855 | LR: 6.56e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.1627s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank3 [2026-10-29 02:24:40,426] INFO : Step 21300/50000 rank3 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0842 | Regression_eta: 0.0878 | Regression_pt: 0.0860 | Regression_sin_phi: 0.1060 | LR: 6.54e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1531s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank3 [2026-10-29 02:28:35,146] INFO : Step 21400/50000 rank3 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.2949 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0942 | Regression_eta: 0.0864 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0843 | LR: 6.51e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.4157s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank3 [2026-10-29 02:32:39,449] INFO : Step 21500/50000 rank3 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0751 | Regression_eta: 0.0532 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0632 | LR: 6.48e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7854s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank3 [2026-10-29 02:36:35,117] INFO : Step 21600/50000 rank3 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2117 | Regression_cos_phi: 0.0596 | Regression_energy: 0.1119 | Regression_eta: 0.1055 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1214 | LR: 6.45e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.8438s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank3 [2026-10-29 02:40:30,747] INFO : Step 21700/50000 rank3 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2659 | Regression_cos_phi: 0.1067 | Regression_energy: 0.0924 | Regression_eta: 0.0825 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0997 | LR: 6.43e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.5966s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank3 [2026-10-29 02:44:20,671] INFO : Step 21800/50000 rank3 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2458 | Regression_cos_phi: 0.0946 | Regression_energy: 0.1078 | Regression_eta: 0.0962 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0954 | LR: 6.40e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.1217s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank3 [2026-10-29 02:48:12,872] INFO : Step 21900/50000 rank3 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1899 | Regression_cos_phi: 0.1065 | Regression_energy: 0.1023 | Regression_eta: 0.0740 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1088 | LR: 6.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2317s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank3 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank3 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2551 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0893 | Regression_eta: 0.0725 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0886 | LR: 6.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1555s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank3 [2026-10-29 02:52:09,890] INFO : Running validation on rank3 +rank3 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:52:57,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:52:58,579] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 02:53:25,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:53:27,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:53:28,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:53:29,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:53:39,517] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 02:53:56,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:53:57,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:53:58,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:53:59,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:54:09,516] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 02:54:10,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:54:10,678] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 02:54:26,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:54:27,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:54:27,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:54:28,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:54:38,558] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank3 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2602 | Regression_cos_phi: 0.1162 | Regression_energy: 0.0908 | Regression_eta: 0.0791 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0895 | LR: 6.32e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4388s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank3 [2026-10-29 03:02:36,686] INFO : Step 22200/50000 rank3 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2029 | Regression_cos_phi: 0.1103 | Regression_energy: 0.0996 | Regression_eta: 0.0915 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1234 | LR: 6.29e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.6556s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank3 [2026-10-29 03:06:28,835] INFO : Step 22300/50000 rank3 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2278 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0838 | Regression_eta: 0.0789 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0859 | LR: 6.26e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8527s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank3 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank3 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2842 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0914 | Regression_eta: 0.0910 | Regression_pt: 0.0930 | Regression_sin_phi: 0.1113 | LR: 6.23e-04 | DataLoad Time: 0.0120s | Model Forward Time: 4.5886s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank3 [2026-10-29 03:14:16,070] INFO : Step 22500/50000 rank3 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2267 | Regression_cos_phi: 0.1175 | Regression_energy: 0.1149 | Regression_eta: 0.0872 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1176 | LR: 6.20e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9004s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank3 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank3 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0974 | Regression_eta: 0.0753 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0934 | LR: 6.18e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5796s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank3 [2026-10-29 03:22:04,589] INFO : Step 22700/50000 rank3 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2501 | Regression_cos_phi: 0.1267 | Regression_energy: 0.0995 | Regression_eta: 0.0884 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0834 | LR: 6.15e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3277s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank3 [2026-10-29 03:26:06,869] INFO : Step 22800/50000 rank3 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2675 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0952 | Regression_eta: 0.0845 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0981 | LR: 6.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2784s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank3 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank3 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0980 | Regression_eta: 0.0656 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0766 | LR: 6.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.6097s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank3 [2026-10-29 03:34:03,730] INFO : Step 23000/50000 rank3 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.3009 | Regression_cos_phi: 0.1129 | Regression_energy: 0.1069 | Regression_eta: 0.0998 | Regression_pt: 0.1119 | Regression_sin_phi: 0.1014 | LR: 6.06e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6400s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank3 [2026-10-29 03:34:04,743] INFO : Running validation on rank3 +rank3 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 03:34:52,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:34:53,013] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 03:35:18,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 03:35:19,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 03:35:20,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 03:35:21,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 03:35:31,704] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:35:32,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 03:35:48,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 03:35:49,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 03:35:50,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 03:35:51,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 03:36:01,667] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 03:36:02,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:36:02,911] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 03:36:18,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 03:36:19,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 03:36:20,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 03:36:20,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 03:36:30,917] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 03:40:33,162] INFO : Step 23100/50000 rank3 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2501 | Regression_cos_phi: 0.1083 | Regression_energy: 0.0953 | Regression_eta: 0.0805 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0816 | LR: 6.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4138s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank3 [2026-10-29 03:44:14,417] INFO : Step 23200/50000 rank3 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0895 | Regression_eta: 0.0623 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0707 | LR: 6.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4838s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank3 [2026-10-29 03:47:55,308] INFO : Step 23300/50000 rank3 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2300 | Regression_cos_phi: 0.1101 | Regression_energy: 0.1160 | Regression_eta: 0.1061 | Regression_pt: 0.1202 | Regression_sin_phi: 0.1142 | LR: 5.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9633s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank3 [2026-10-29 03:51:33,797] INFO : Step 23400/50000 rank3 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0908 | Classification_binary: 0.2536 | Regression_cos_phi: 0.1060 | Regression_energy: 0.0951 | Regression_eta: 0.1040 | Regression_pt: 0.0981 | Regression_sin_phi: 0.1177 | LR: 5.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1980s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank3 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank3 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2435 | Regression_cos_phi: 0.1035 | Regression_energy: 0.0959 | Regression_eta: 0.0745 | Regression_pt: 0.0988 | Regression_sin_phi: 0.1117 | LR: 5.92e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0141s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank3 [2026-10-29 03:59:15,167] INFO : Step 23600/50000 rank3 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2113 | Regression_cos_phi: 0.1080 | Regression_energy: 0.1074 | Regression_eta: 0.0894 | Regression_pt: 0.1126 | Regression_sin_phi: 0.1061 | LR: 5.90e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.9374s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank3 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank3 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2373 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0843 | Regression_eta: 0.0774 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0972 | LR: 5.87e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3851s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank3 [2026-10-29 04:06:50,322] INFO : Step 23800/50000 rank3 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2496 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1118 | Regression_eta: 0.0987 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0957 | LR: 5.84e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.0040s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank3 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank3 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0895 | Regression_eta: 0.0587 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0983 | LR: 5.81e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3358s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank3 [2026-10-29 04:14:57,721] INFO : Step 24000/50000 rank3 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0957 | Regression_energy: 0.1050 | Regression_eta: 0.0842 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0941 | LR: 5.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1442s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank3 [2026-10-29 04:14:58,739] INFO : Running validation on rank3 +rank3 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:15:47,909] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:15:48,539] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 04:16:15,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:16:16,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:16:17,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:16:18,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:16:28,399] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:16:29,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 04:16:45,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:16:46,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:16:47,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:16:48,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:16:58,223] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:16:59,584] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 04:17:15,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:17:15,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:17:16,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:17:17,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:17:27,317] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 04:21:44,038] INFO : Step 24100/50000 rank3 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2682 | Regression_cos_phi: 0.1147 | Regression_energy: 0.0997 | Regression_eta: 0.0774 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0932 | LR: 5.75e-04 | DataLoad Time: 0.0203s | Model Forward Time: 7.7520s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank3 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank3 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0842 | Regression_energy: 0.1063 | Regression_eta: 0.0772 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0676 | LR: 5.73e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2944s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank3 [2026-10-29 04:29:22,271] INFO : Step 24300/50000 rank3 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2162 | Regression_cos_phi: 0.0926 | Regression_energy: 0.1152 | Regression_eta: 0.0749 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0755 | LR: 5.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0423s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank3 [2026-10-29 04:33:07,494] INFO : Step 24400/50000 rank3 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2767 | Regression_cos_phi: 0.1154 | Regression_energy: 0.1335 | Regression_eta: 0.0933 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0961 | LR: 5.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2377s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank3 [2026-10-29 04:36:57,190] INFO : Step 24500/50000 rank3 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0891 | Regression_energy: 0.1129 | Regression_eta: 0.0605 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0660 | LR: 5.64e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3261s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank3 [2026-10-29 04:40:40,562] INFO : Step 24600/50000 rank3 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2060 | Regression_cos_phi: 0.1259 | Regression_energy: 0.1199 | Regression_eta: 0.1172 | Regression_pt: 0.1229 | Regression_sin_phi: 0.1033 | LR: 5.61e-04 | DataLoad Time: 0.0262s | Model Forward Time: 3.9892s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank3 [2026-10-29 04:44:29,989] INFO : Step 24700/50000 rank3 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1015 | Classification_binary: 0.2466 | Regression_cos_phi: 0.1786 | Regression_energy: 0.1207 | Regression_eta: 0.1252 | Regression_pt: 0.1294 | Regression_sin_phi: 0.1342 | LR: 5.58e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5088s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank3 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank3 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2676 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0899 | Regression_eta: 0.1056 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0956 | LR: 5.56e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8644s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank3 [2026-10-29 04:52:00,597] INFO : Step 24900/50000 rank3 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.2121 | Regression_cos_phi: 0.1298 | Regression_energy: 0.1052 | Regression_eta: 0.0785 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1209 | LR: 5.53e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9783s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank3 [2026-10-29 04:55:44,333] INFO : Step 25000/50000 rank3 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0945 | Regression_energy: 0.1104 | Regression_eta: 0.0691 | Regression_pt: 0.1147 | Regression_sin_phi: 0.1042 | LR: 5.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2484s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank3 [2026-10-29 04:55:45,304] INFO : Running validation on rank3 +rank3 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:56:33,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:56:34,566] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 04:57:00,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:57:01,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:57:02,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:57:02,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:57:12,946] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:57:13,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:57:13,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 04:57:29,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:57:30,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:57:31,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:57:32,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:57:42,497] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:57:44,416] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 04:58:00,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:58:00,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:58:01,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:58:02,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:58:12,180] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 05:02:15,912] INFO : Step 25100/50000 rank3 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2742 | Regression_cos_phi: 0.1058 | Regression_energy: 0.0926 | Regression_eta: 0.0737 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0790 | LR: 5.47e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.7065s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank3 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank3 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2162 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0947 | Regression_eta: 0.0797 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0921 | LR: 5.44e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2807s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank3 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank3 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0855 | Regression_energy: 0.1224 | Regression_eta: 0.0676 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0719 | LR: 5.42e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8564s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank3 [2026-10-29 05:13:35,656] INFO : Step 25400/50000 rank3 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0920 | Classification_binary: 0.3044 | Regression_cos_phi: 0.1323 | Regression_energy: 0.1174 | Regression_eta: 0.1162 | Regression_pt: 0.1218 | Regression_sin_phi: 0.1197 | LR: 5.39e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0684s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank3 [2026-10-29 05:17:27,535] INFO : Step 25500/50000 rank3 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2816 | Regression_cos_phi: 0.0951 | Regression_energy: 0.1022 | Regression_eta: 0.0860 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0834 | LR: 5.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0762s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank3 [2026-10-29 05:21:16,672] INFO : Step 25600/50000 rank3 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2242 | Regression_cos_phi: 0.0986 | Regression_energy: 0.1122 | Regression_eta: 0.0828 | Regression_pt: 0.1166 | Regression_sin_phi: 0.1109 | LR: 5.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8769s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank3 [2026-10-29 05:25:00,204] INFO : Step 25700/50000 rank3 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2395 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1067 | Regression_eta: 0.0621 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0894 | LR: 5.30e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8866s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank3 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank3 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1012 | Classification_binary: 0.3043 | Regression_cos_phi: 0.1235 | Regression_energy: 0.1282 | Regression_eta: 0.0906 | Regression_pt: 0.1331 | Regression_sin_phi: 0.1131 | LR: 5.27e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5172s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank3 [2026-10-29 05:32:33,909] INFO : Step 25900/50000 rank3 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2025 | Regression_cos_phi: 0.0958 | Regression_energy: 0.1106 | Regression_eta: 0.0681 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0602 | LR: 5.25e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8724s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank3 [2026-10-29 05:36:28,636] INFO : Step 26000/50000 rank3 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.2222 | Regression_cos_phi: 0.1208 | Regression_energy: 0.1223 | Regression_eta: 0.1262 | Regression_pt: 0.1250 | Regression_sin_phi: 0.1161 | LR: 5.22e-04 | DataLoad Time: 0.0176s | Model Forward Time: 9.2339s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank3 [2026-10-29 05:36:29,552] INFO : Running validation on rank3 +rank3 [2026-10-29 05:36:29,552] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 05:37:19,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:37:20,019] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 05:37:45,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 05:37:46,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 05:37:46,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 05:37:47,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 05:37:57,912] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 05:37:58,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 05:38:15,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 05:38:16,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 05:38:17,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 05:38:18,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 05:38:28,105] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 05:38:28,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:38:28,893] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 05:38:44,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 05:38:44,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 05:38:45,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 05:38:46,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 05:38:56,397] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 05:43:04,442] INFO : Step 26100/50000 rank3 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2668 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0984 | Regression_eta: 0.0725 | Regression_pt: 0.1029 | Regression_sin_phi: 0.0961 | LR: 5.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8571s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank3 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank3 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0703 | Regression_energy: 0.1091 | Regression_eta: 0.0592 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0679 | LR: 5.16e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2718s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank3 [2026-10-29 05:50:52,484] INFO : Step 26300/50000 rank3 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0759 | Regression_energy: 0.1114 | Regression_eta: 0.0469 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0576 | LR: 5.13e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3834s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank3 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank3 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2484 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1486 | Regression_eta: 0.0896 | Regression_pt: 0.1555 | Regression_sin_phi: 0.1027 | LR: 5.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3824s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank3 [2026-10-29 05:58:42,121] INFO : Step 26500/50000 rank3 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2860 | Regression_cos_phi: 0.0895 | Regression_energy: 0.1228 | Regression_eta: 0.0680 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0931 | LR: 5.08e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4053s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank3 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank3 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0749 | Regression_energy: 0.1145 | Regression_eta: 0.0714 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0774 | LR: 5.05e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.7816s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank3 [2026-10-29 06:06:39,601] INFO : Step 26700/50000 rank3 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.2398 | Regression_cos_phi: 0.1094 | Regression_energy: 0.1232 | Regression_eta: 0.1104 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0959 | LR: 5.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.6715s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank3 [2026-10-29 06:10:41,255] INFO : Step 26800/50000 rank3 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0947 | Classification_binary: 0.3391 | Regression_cos_phi: 0.1327 | Regression_energy: 0.1313 | Regression_eta: 0.1021 | Regression_pt: 0.1378 | Regression_sin_phi: 0.1075 | LR: 4.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2408s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank3 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank3 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0700 | Regression_energy: 0.1340 | Regression_eta: 0.0544 | Regression_pt: 0.1429 | Regression_sin_phi: 0.0504 | LR: 4.96e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.8907s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank3 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank3 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2160 | Regression_cos_phi: 0.1190 | Regression_energy: 0.1135 | Regression_eta: 0.0769 | Regression_pt: 0.1188 | Regression_sin_phi: 0.1130 | LR: 4.94e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9886s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank3 [2026-10-29 06:18:43,932] INFO : Running validation on rank3 +rank3 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 06:19:32,614] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:19:33,153] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 06:20:00,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 06:20:01,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 06:20:02,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 06:20:03,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 06:20:13,507] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:20:14,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:20:14,781] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 06:20:30,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 06:20:31,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 06:20:32,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 06:20:33,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 06:20:43,925] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:20:45,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:20:46,292] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 06:21:02,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 06:21:02,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 06:21:03,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 06:21:04,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 06:21:14,407] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 06:25:28,750] INFO : Step 27100/50000 rank3 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2690 | Regression_cos_phi: 0.1015 | Regression_energy: 0.1231 | Regression_eta: 0.0727 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0993 | LR: 4.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7016s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank3 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank3 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0809 | Regression_energy: 0.1315 | Regression_eta: 0.0927 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0787 | LR: 4.88e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.5740s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank3 [2026-10-29 06:33:36,120] INFO : Step 27300/50000 rank3 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0977 | Regression_energy: 0.1261 | Regression_eta: 0.0662 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0906 | LR: 4.85e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.9961s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank3 [2026-10-29 06:37:36,742] INFO : Step 27400/50000 rank3 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2223 | Regression_cos_phi: 0.0980 | Regression_energy: 0.1028 | Regression_eta: 0.0718 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1092 | LR: 4.82e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5113s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank3 [2026-10-29 06:41:28,998] INFO : Step 27500/50000 rank3 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0924 | Regression_energy: 0.1353 | Regression_eta: 0.0630 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0770 | LR: 4.80e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9228s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank3 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank3 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0944 | Regression_energy: 0.1168 | Regression_eta: 0.0744 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0964 | LR: 4.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8531s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank3 [2026-10-29 06:49:16,009] INFO : Step 27700/50000 rank3 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2059 | Regression_cos_phi: 0.0799 | Regression_energy: 0.1444 | Regression_eta: 0.0762 | Regression_pt: 0.1513 | Regression_sin_phi: 0.0958 | LR: 4.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0200s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank3 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank3 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1166 | Regression_energy: 0.1184 | Regression_eta: 0.0960 | Regression_pt: 0.1241 | Regression_sin_phi: 0.1106 | LR: 4.71e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.6692s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank3 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank3 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2357 | Regression_cos_phi: 0.1235 | Regression_energy: 0.1110 | Regression_eta: 0.0884 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1410 | LR: 4.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5354s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank3 [2026-10-29 07:01:10,307] INFO : Step 28000/50000 rank3 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0709 | Regression_energy: 0.1372 | Regression_eta: 0.0832 | Regression_pt: 0.1408 | Regression_sin_phi: 0.1012 | LR: 4.66e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3451s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank3 [2026-10-29 07:01:11,148] INFO : Running validation on rank3 +rank3 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:01:11,148] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:02:01,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:02:02,348] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 07:02:30,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:02:30,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:02:31,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:02:32,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:02:42,856] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:02:44,046] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 07:03:00,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:03:01,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:03:02,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:03:03,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:03:13,667] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:03:14,473] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 07:03:30,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:03:30,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:03:31,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:03:32,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:03:42,210] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 07:08:04,579] INFO : Step 28100/50000 rank3 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2374 | Regression_cos_phi: 0.0993 | Regression_energy: 0.1433 | Regression_eta: 0.0652 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0848 | LR: 4.63e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1498s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank3 [2026-10-29 07:11:56,589] INFO : Step 28200/50000 rank3 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2403 | Regression_cos_phi: 0.0884 | Regression_energy: 0.1182 | Regression_eta: 0.0622 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0974 | LR: 4.60e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2159s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank3 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank3 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0757 | Regression_energy: 0.1289 | Regression_eta: 0.0604 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0725 | LR: 4.57e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8321s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank3 [2026-10-29 07:20:03,894] INFO : Step 28400/50000 rank3 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1909 | Regression_cos_phi: 0.1181 | Regression_energy: 0.1198 | Regression_eta: 0.0852 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0983 | LR: 4.55e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0883s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank3 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank3 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2561 | Regression_cos_phi: 0.0593 | Regression_energy: 0.1184 | Regression_eta: 0.0745 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0946 | LR: 4.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1335s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank3 [2026-10-29 07:27:52,534] INFO : Step 28600/50000 rank3 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0992 | Regression_energy: 0.0972 | Regression_eta: 0.0710 | Regression_pt: 0.1023 | Regression_sin_phi: 0.1039 | LR: 4.49e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2890s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank3 [2026-10-29 07:31:49,173] INFO : Step 28700/50000 rank3 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0983 | Regression_energy: 0.1129 | Regression_eta: 0.0761 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0989 | LR: 4.46e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0946s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank3 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank3 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2149 | Regression_cos_phi: 0.1010 | Regression_energy: 0.1196 | Regression_eta: 0.0752 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0890 | LR: 4.44e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank3 [2026-10-29 07:39:38,710] INFO : Step 28900/50000 rank3 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2088 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1009 | Regression_eta: 0.0503 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0761 | LR: 4.41e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3187s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank3 [2026-10-29 07:43:43,866] INFO : Step 29000/50000 rank3 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0676 | Regression_energy: 0.1097 | Regression_eta: 0.0674 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0687 | LR: 4.38e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9074s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank3 [2026-10-29 07:43:44,819] INFO : Running validation on rank3 +rank3 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:44:33,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:44:33,717] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 07:44:59,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:45:00,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:45:01,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:45:02,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:45:12,341] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:45:12,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:45:13,376] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 07:45:29,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:45:30,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:45:31,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:45:32,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:45:42,313] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:45:43,719] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 07:45:59,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:46:00,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:46:00,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:46:01,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:46:11,613] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 07:50:42,138] INFO : Step 29100/50000 rank3 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2688 | Regression_cos_phi: 0.1021 | Regression_energy: 0.1244 | Regression_eta: 0.0760 | Regression_pt: 0.1308 | Regression_sin_phi: 0.1152 | LR: 4.35e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.6994s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank3 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank3 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0885 | Regression_eta: 0.0524 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0519 | LR: 4.33e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2678s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank3 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank3 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0843 | Regression_energy: 0.1031 | Regression_eta: 0.0521 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0702 | LR: 4.30e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9991s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank3 [2026-10-29 08:02:11,407] INFO : Step 29400/50000 rank3 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0762 | Regression_energy: 0.1022 | Regression_eta: 0.0580 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0649 | LR: 4.27e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5190s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank3 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank3 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2659 | Regression_cos_phi: 0.1224 | Regression_energy: 0.1000 | Regression_eta: 0.0925 | Regression_pt: 0.1059 | Regression_sin_phi: 0.1066 | LR: 4.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4991s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank3 [2026-10-29 08:09:49,625] INFO : Step 29600/50000 rank3 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0944 | Regression_eta: 0.0608 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0794 | LR: 4.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7090s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank3 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank3 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0870 | Regression_eta: 0.0484 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0615 | LR: 4.19e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1850s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank3 [2026-10-29 08:17:24,689] INFO : Step 29800/50000 rank3 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0872 | Regression_eta: 0.0631 | Regression_pt: 0.0915 | Regression_sin_phi: 0.1012 | LR: 4.16e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3846s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank3 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank3 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0705 | Regression_energy: 0.1020 | Regression_eta: 0.0607 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0863 | LR: 4.14e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9951s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank3 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank3 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0912 | Regression_eta: 0.0581 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0720 | LR: 4.11e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7040s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank3 [2026-10-29 08:24:57,340] INFO : Running validation on rank3 +rank3 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 08:25:45,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:25:45,651] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 08:26:11,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 08:26:12,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 08:26:13,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 08:26:14,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 08:26:24,232] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 08:26:24,835] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:26:25,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 08:26:41,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 08:26:42,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 08:26:43,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 08:26:44,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 08:26:54,307] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:26:55,056] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 08:27:11,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 08:27:11,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 08:27:12,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 08:27:13,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 08:27:23,112] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank3 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2481 | Regression_cos_phi: 0.0797 | Regression_energy: 0.1009 | Regression_eta: 0.0697 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0960 | LR: 4.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank3 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank3 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0877 | Regression_eta: 0.0615 | Regression_pt: 0.0920 | Regression_sin_phi: 0.0876 | LR: 4.06e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5339s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank3 [2026-10-29 08:39:30,453] INFO : Step 30300/50000 rank3 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1935 | Regression_cos_phi: 0.1025 | Regression_energy: 0.1036 | Regression_eta: 0.0562 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0738 | LR: 4.03e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1807s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank3 [2026-10-29 08:43:19,168] INFO : Step 30400/50000 rank3 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2481 | Regression_cos_phi: 0.0877 | Regression_energy: 0.1095 | Regression_eta: 0.0722 | Regression_pt: 0.1128 | Regression_sin_phi: 0.1043 | LR: 4.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5059s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank3 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank3 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0877 | Regression_eta: 0.0679 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0811 | LR: 3.98e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7630s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank3 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank3 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0924 | Regression_eta: 0.0570 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0634 | LR: 3.95e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank3 [2026-10-29 08:54:35,590] INFO : Step 30700/50000 rank3 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0754 | Regression_eta: 0.0635 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0629 | LR: 3.92e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2961s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank3 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank3 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0912 | Regression_energy: 0.1022 | Regression_eta: 0.0681 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0805 | LR: 3.90e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2069s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank3 [2026-10-29 09:02:09,796] INFO : Step 30900/50000 rank3 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2126 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0869 | Regression_eta: 0.0599 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0853 | LR: 3.87e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9282s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank3 [2026-10-29 09:05:52,366] INFO : Step 31000/50000 rank3 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0831 | Regression_eta: 0.0569 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0438 | LR: 3.84e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.1881s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank3 [2026-10-29 09:05:53,352] INFO : Running validation on rank3 +rank3 [2026-10-29 09:05:53,352] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:05:53,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:06:41,697] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:06:42,226] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 09:07:08,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:07:09,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:07:10,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:07:11,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:07:21,247] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:07:22,582] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 09:07:38,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:07:39,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:07:40,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:07:41,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:07:51,549] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 09:07:51,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:07:52,508] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 09:08:08,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:08:09,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:08:09,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:08:10,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:08:20,691] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank3 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2026 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0880 | Regression_eta: 0.0685 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0874 | LR: 3.82e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4607s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank3 [2026-10-29 09:16:29,207] INFO : Step 31200/50000 rank3 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2878 | Regression_cos_phi: 0.0865 | Regression_energy: 0.1024 | Regression_eta: 0.0799 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0908 | LR: 3.79e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.5362s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank3 [2026-10-29 09:20:24,831] INFO : Step 31300/50000 rank3 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0734 | Regression_eta: 0.0539 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0491 | LR: 3.76e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.4703s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank3 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank3 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0917 | Regression_eta: 0.0616 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0761 | LR: 3.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8366s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank3 [2026-10-29 09:28:17,179] INFO : Step 31500/50000 rank3 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0825 | Regression_eta: 0.0714 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0640 | LR: 3.71e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3621s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank3 [2026-10-29 09:32:16,798] INFO : Step 31600/50000 rank3 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0962 | Regression_eta: 0.0712 | Regression_pt: 0.0995 | Regression_sin_phi: 0.1058 | LR: 3.69e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.8619s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank3 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank3 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0732 | Regression_eta: 0.0489 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0665 | LR: 3.66e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.1347s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank3 [2026-10-29 09:40:22,622] INFO : Step 31800/50000 rank3 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0769 | Regression_eta: 0.0535 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0827 | LR: 3.64e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3537s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank3 [2026-10-29 09:48:15,344] INFO : Step 31900/50000 rank3 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0836 | Regression_eta: 0.0543 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0750 | LR: 3.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8705s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank3 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank3 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0802 | Regression_eta: 0.0562 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0685 | LR: 3.58e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1227s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank3 [2026-10-29 09:52:26,791] INFO : Running validation on rank3 +rank3 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:53:17,424] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 09:53:45,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:53:47,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:53:48,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:53:49,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:53:59,566] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:54:01,009] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 09:54:17,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:54:18,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:54:19,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:54:20,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:54:30,282] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:54:30,788] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 09:54:46,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:54:46,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:54:47,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:54:48,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:54:58,308] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank3 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.1870 | Regression_cos_phi: 0.0686 | Regression_energy: 0.1042 | Regression_eta: 0.0544 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0833 | LR: 3.56e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.9455s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank3 [2026-10-29 10:03:26,459] INFO : Step 32200/50000 rank3 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2631 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0808 | Regression_eta: 0.0705 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0644 | LR: 3.53e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5228s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank3 [2026-10-29 10:07:22,791] INFO : Step 32300/50000 rank3 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0789 | Regression_eta: 0.0915 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0973 | LR: 3.51e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5872s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank3 [2026-10-29 10:11:42,985] INFO : Step 32400/50000 rank3 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0745 | Regression_eta: 0.0759 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0780 | LR: 3.48e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.0249s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank3 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank3 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.2390 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0760 | Regression_eta: 0.0750 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0659 | LR: 3.46e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5249s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank3 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank3 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0877 | Regression_eta: 0.0581 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0647 | LR: 3.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4752s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank3 [2026-10-29 10:24:11,750] INFO : Step 32700/50000 rank3 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0927 | Regression_eta: 0.0525 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0784 | LR: 3.41e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0514s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank3 [2026-10-29 10:28:05,809] INFO : Step 32800/50000 rank3 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0777 | Regression_eta: 0.0737 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0660 | LR: 3.38e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5517s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank3 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank3 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2329 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0835 | Regression_eta: 0.0672 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0764 | LR: 3.36e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8094s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank3 [2026-10-29 10:36:14,933] INFO : Step 33000/50000 rank3 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0677 | Regression_eta: 0.0424 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0577 | LR: 3.33e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7651s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank3 [2026-10-29 10:36:15,962] INFO : Running validation on rank3 +rank3 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 10:37:03,787] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:37:04,370] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 10:37:32,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 10:37:33,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 10:37:35,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 10:37:37,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 10:37:47,190] INFO : Time taken to make predictions on device 3 is: 0.29 min +rank3 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:37:48,815] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 10:38:05,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 10:38:06,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 10:38:07,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 10:38:08,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 10:38:18,353] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:38:21,705] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 10:38:37,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 10:38:38,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 10:38:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 10:38:39,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 10:38:49,583] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 10:43:12,059] INFO : Step 33100/50000 rank3 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0749 | Regression_eta: 0.0445 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0640 | LR: 3.31e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9496s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank3 [2026-10-29 10:47:22,843] INFO : Step 33200/50000 rank3 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0894 | Regression_eta: 0.0595 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0514 | LR: 3.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6964s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank3 [2026-10-29 10:51:33,084] INFO : Step 33300/50000 rank3 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0921 | Regression_eta: 0.0589 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0586 | LR: 3.26e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.4494s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank3 [2026-10-29 10:55:47,786] INFO : Step 33400/50000 rank3 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0775 | Regression_eta: 0.0691 | Regression_pt: 0.0796 | Regression_sin_phi: 0.1077 | LR: 3.23e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5464s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank3 [2026-10-29 10:59:48,622] INFO : Step 33500/50000 rank3 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0928 | Classification_binary: 0.2561 | Regression_cos_phi: 0.1042 | Regression_energy: 0.0848 | Regression_eta: 0.0838 | Regression_pt: 0.0882 | Regression_sin_phi: 0.1012 | LR: 3.21e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.9158s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank3 [2026-10-29 11:05:24,496] INFO : Step 33600/50000 rank3 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2326 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0952 | Regression_eta: 0.0607 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0776 | LR: 3.19e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.2025s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank3 [2026-10-29 11:09:08,679] INFO : Step 33700/50000 rank3 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0804 | Regression_eta: 0.0437 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0620 | LR: 3.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3172s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank3 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank3 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0891 | Regression_eta: 0.0645 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0765 | LR: 3.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3671s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank3 [2026-10-29 11:16:52,540] INFO : Step 33900/50000 rank3 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0784 | Regression_eta: 0.0610 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0790 | LR: 3.11e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.8118s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank3 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank3 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0748 | Regression_eta: 0.0538 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0754 | LR: 3.09e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.9492s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank3 [2026-10-29 11:20:48,852] INFO : Running validation on rank3 +rank3 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 11:21:42,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:21:42,910] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 11:22:09,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 11:22:10,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 11:22:11,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 11:22:11,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 11:22:21,966] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:22:22,977] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 11:22:39,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 11:22:40,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 11:22:41,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 11:22:42,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 11:22:52,138] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:22:53,477] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 11:23:09,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 11:23:10,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 11:23:11,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 11:23:11,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 11:23:21,741] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 11:27:42,516] INFO : Step 34100/50000 rank3 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0797 | Regression_eta: 0.0565 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0636 | LR: 3.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.5168s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank3 [2026-10-29 11:31:45,285] INFO : Step 34200/50000 rank3 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2355 | Regression_cos_phi: 0.1193 | Regression_energy: 0.0804 | Regression_eta: 0.1068 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0837 | LR: 3.04e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7877s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank3 [2026-10-29 11:35:56,810] INFO : Step 34300/50000 rank3 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0647 | Regression_eta: 0.0565 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0803 | LR: 3.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.2297s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank3 [2026-10-29 11:40:08,374] INFO : Step 34400/50000 rank3 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1735 | Regression_cos_phi: 0.1065 | Regression_energy: 0.0833 | Regression_eta: 0.0586 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0922 | LR: 2.99e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.9081s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank3 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank3 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2226 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0965 | Regression_eta: 0.0539 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0625 | LR: 2.97e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5071s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank3 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank3 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2305 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0706 | Regression_eta: 0.0573 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0822 | LR: 2.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6059s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank3 [2026-10-29 11:52:00,067] INFO : Step 34700/50000 rank3 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0855 | Regression_eta: 0.0536 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0629 | LR: 2.92e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.5509s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank3 [2026-10-29 11:55:53,802] INFO : Step 34800/50000 rank3 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0725 | Regression_eta: 0.0378 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0630 | LR: 2.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8584s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank3 [2026-10-29 11:59:58,708] INFO : Step 34900/50000 rank3 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0900 | Regression_eta: 0.0473 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0627 | LR: 2.88e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1712s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank3 [2026-10-29 12:04:06,317] INFO : Step 35000/50000 rank3 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0753 | Regression_eta: 0.0551 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0734 | LR: 2.85e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4572s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank3 [2026-10-29 12:04:07,347] INFO : Running validation on rank3 +rank3 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 12:05:04,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:05:05,387] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 12:05:31,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 12:05:32,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 12:05:33,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 12:05:34,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 12:05:44,756] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 12:06:01,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 12:06:02,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 12:06:03,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 12:06:04,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 12:06:14,522] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 12:06:15,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:06:16,077] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 12:06:32,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 12:06:32,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 12:06:33,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 12:06:34,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 12:06:44,233] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank3 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0786 | Regression_eta: 0.0696 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0628 | LR: 2.83e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.6532s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank3 [2026-10-29 12:15:06,104] INFO : Step 35200/50000 rank3 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2221 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0821 | Regression_eta: 0.0746 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0783 | LR: 2.81e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0447s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank3 [2026-10-29 12:18:57,447] INFO : Step 35300/50000 rank3 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0911 | Regression_eta: 0.0687 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0749 | LR: 2.79e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8172s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank3 [2026-10-29 12:23:00,203] INFO : Step 35400/50000 rank3 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0946 | Regression_eta: 0.1097 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0691 | LR: 2.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3962s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank3 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank3 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0793 | Regression_eta: 0.0570 | Regression_pt: 0.0809 | Regression_sin_phi: 0.1221 | LR: 2.74e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2683s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank3 [2026-10-29 12:31:17,345] INFO : Step 35600/50000 rank3 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2578 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0744 | Regression_eta: 0.0753 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0578 | LR: 2.72e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1676s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank3 [2026-10-29 12:35:31,256] INFO : Step 35700/50000 rank3 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0676 | Regression_eta: 0.0612 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0985 | LR: 2.70e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2788s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank3 [2026-10-29 14:49:26,299] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-29 14:49:26,322] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank3 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-29 14:49:53,253] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank3 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank3 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank3 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank3 [2026-10-29 14:49:53,280] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,283] INFO : Initializing backbone layer 3 +rank3 [2026-10-29 14:49:53,283] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,286] INFO : Initializing backbone layer 4 +rank3 [2026-10-29 14:49:53,286] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,289] INFO : Initializing backbone layer 5 +rank3 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank3 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank3 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank3 [2026-10-29 14:49:53,342] INFO : Output DNNs initialization took 0.05s +rank3 [2026-10-29 14:49:53,342] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-29 14:49:53,342] INFO : jagged_attention enabled=True +rank3 [2026-10-29 14:49:53,342] INFO : flash_attn_varlen enabled=True +rank3 [2026-10-29 14:49:53,342] INFO : task_queries enabled=False +rank3 [2026-10-29 14:49:53,342] INFO : classification_norm parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : regression_norm parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : classification_query parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank3 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank3 [2026-10-29 14:49:53,343] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-29 14:49:53,343] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_pid parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_pt parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_eta parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-29 14:49:53,343] INFO : nn_energy parameters: 0 +rank3 [2026-10-29 14:49:53,344] INFO : Total MLPF parameters: 7260244 +rank3 [2026-10-29 14:49:53,344] INFO : MLPF __init__ done +rank3 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank3 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=3 +rank3 [2026-10-29 14:49:53,362] INFO : Moved model to device rank=3 +rank3 [2026-10-29 14:49:53,364] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-29 14:49:53,694] INFO : Restoring training from step 35001 +rank3 [2026-10-29 14:49:53,700] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank3 [2026-10-29 14:49:53,701] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-29 14:49:53,913] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-29 14:49:54,030] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank3 [2026-10-29 14:49:54,031] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-29 14:49:54,032] INFO : Trainable parameters: 7260244 +rank3 [2026-10-29 14:49:54,032] INFO : Non-trainable parameters: 0 +rank3 [2026-10-29 14:49:54,032] INFO : Total parameters: 7260244 +rank3 [2026-10-29 14:49:54,035] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-29 14:49:54,035] INFO : Getting dataloaders +rank3 [2026-10-29 14:49:54,035] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:01,390] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:02,341] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:03,545] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-29 14:50:03,545] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-29 14:50:03,545] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:03,810] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:04,057] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:04,390] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-29 14:50:04,390] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-29 14:50:17,813] INFO : Got dataloaders +rank3 [2026-10-29 14:50:17,813] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank3 [2026-10-29 14:50:17,813] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank3 [2026-10-29 14:50:17,813] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank3 [2026-10-29 14:50:17,813] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank3 [2026-10-29 14:50:19,613] INFO : loader split=train rank=3 len=21108 +rank3 [2026-10-29 14:50:19,613] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-29 14:50:19,613] INFO : Starting training from step 35001 to 50000 on rank 3 of 8 +rank3 [2026-10-29 14:50:19,613] INFO : Creating train_iterator +rank3 [2026-10-29 14:50:19,613] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank3 [2026-10-29 14:50:34,018] INFO : Created train_iterator +rank3 [2026-10-29 14:54:16,363] INFO : Step 35100/50000 rank3 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0798 | Regression_eta: 0.0815 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0827 | LR: 2.83e-04 | DataLoad Time: 0.0167s | Model Forward Time: 2.8676s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank3 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank3 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0691 | Regression_eta: 0.0720 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0719 | LR: 2.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5089s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank3 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank3 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0794 | Regression_eta: 0.0613 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0709 | LR: 2.79e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5911s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank3 [2026-10-29 15:05:10,669] INFO : Step 35400/50000 rank3 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0840 | Regression_eta: 0.1205 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0955 | LR: 2.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4258s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank3 [2026-10-29 15:08:49,316] INFO : Step 35500/50000 rank3 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1796 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0779 | Regression_eta: 0.0493 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0504 | LR: 2.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.4550s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank3 [2026-10-29 15:12:27,419] INFO : Step 35600/50000 rank3 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2499 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0820 | Regression_eta: 0.0519 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0539 | LR: 2.72e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.6892s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank3 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank3 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0890 | Regression_eta: 0.0531 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0883 | LR: 2.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.3534s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank3 [2026-10-29 15:19:47,617] INFO : Step 35800/50000 rank3 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0909 | Regression_eta: 0.0856 | Regression_pt: 0.0957 | Regression_sin_phi: 0.1373 | LR: 2.68e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.4899s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank3 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank3 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0798 | Regression_eta: 0.0563 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0725 | LR: 2.65e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8620s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank3 [2026-10-29 15:27:19,182] INFO : Step 36000/50000 rank3 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0782 | Regression_eta: 0.0598 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0626 | LR: 2.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8287s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank3 [2026-10-29 15:27:20,114] INFO : Running validation on rank3 +rank3 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 15:28:08,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:28:08,878] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 15:28:36,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 15:28:37,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 15:28:38,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 15:28:39,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 15:28:49,432] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 15:28:49,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:28:50,427] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 15:29:06,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 15:29:07,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 15:29:08,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 15:29:09,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 15:29:19,191] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:29:20,197] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 15:29:36,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 15:29:36,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 15:29:37,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 15:29:38,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 15:29:48,372] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank3 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0887 | Regression_eta: 0.0720 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0837 | LR: 2.61e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.9087s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank3 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank3 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2107 | Regression_cos_phi: 0.1513 | Regression_energy: 0.0819 | Regression_eta: 0.0989 | Regression_pt: 0.0863 | Regression_sin_phi: 0.1180 | LR: 2.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5680s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank3 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank3 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1935 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0607 | Regression_eta: 0.0415 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0615 | LR: 2.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5120s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank3 [2026-10-29 15:45:12,808] INFO : Step 36400/50000 rank3 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0906 | Regression_eta: 0.0637 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0480 | LR: 2.55e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.2458s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank3 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank3 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0811 | Regression_eta: 0.0668 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0744 | LR: 2.52e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.0157s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank3 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank3 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2086 | Regression_cos_phi: 0.1161 | Regression_energy: 0.0713 | Regression_eta: 0.0700 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0839 | LR: 2.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7041s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank3 [2026-10-29 15:56:51,300] INFO : Step 36700/50000 rank3 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0976 | Regression_eta: 0.0564 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0733 | LR: 2.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7861s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank3 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank3 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0936 | Regression_eta: 0.0637 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0621 | LR: 2.46e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4175s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank3 [2026-10-29 16:04:39,809] INFO : Step 36900/50000 rank3 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0858 | Regression_eta: 0.0488 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0859 | LR: 2.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank3 [2026-10-29 16:08:30,144] INFO : Step 37000/50000 rank3 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0924 | Regression_eta: 0.0471 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0669 | LR: 2.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9059s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank3 [2026-10-29 16:08:31,022] INFO : Running validation on rank3 +rank3 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:09:20,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:09:20,760] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 16:09:46,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:09:47,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:09:48,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:09:49,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:09:59,064] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 16:10:16,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:10:17,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:10:18,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:10:19,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:10:29,023] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:10:30,423] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 16:10:46,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:10:47,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:10:47,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:10:48,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:10:58,476] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 16:15:06,402] INFO : Step 37100/50000 rank3 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0738 | Regression_eta: 0.0518 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0542 | LR: 2.40e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8502s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank3 [2026-10-29 16:18:46,283] INFO : Step 37200/50000 rank3 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2059 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0815 | Regression_eta: 0.0570 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0666 | LR: 2.38e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2305s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank3 [2026-10-29 16:22:16,728] INFO : Step 37300/50000 rank3 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.2062 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0798 | Regression_eta: 0.0767 | Regression_pt: 0.0827 | Regression_sin_phi: 0.1012 | LR: 2.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5798s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank3 [2026-10-29 16:26:01,825] INFO : Step 37400/50000 rank3 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0866 | Regression_eta: 0.0722 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0748 | LR: 2.34e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9419s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank3 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank3 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0875 | Regression_eta: 0.0596 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0754 | LR: 2.32e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3477s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank3 [2026-10-29 16:34:14,939] INFO : Step 37600/50000 rank3 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2284 | Regression_cos_phi: 0.0864 | Regression_energy: 0.1026 | Regression_eta: 0.0695 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0898 | LR: 2.30e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1115s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank3 [2026-10-29 16:38:00,807] INFO : Step 37700/50000 rank3 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0831 | Regression_eta: 0.0519 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0618 | LR: 2.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1655s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank3 [2026-10-29 16:41:51,411] INFO : Step 37800/50000 rank3 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0785 | Regression_eta: 0.0594 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0678 | LR: 2.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.2932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank3 [2026-10-29 16:45:39,811] INFO : Step 37900/50000 rank3 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0738 | Regression_eta: 0.0515 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0889 | LR: 2.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8069s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank3 [2026-10-29 16:49:32,191] INFO : Step 38000/50000 rank3 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0760 | Regression_eta: 0.0693 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0578 | LR: 2.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8884s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank3 [2026-10-29 16:49:33,136] INFO : Running validation on rank3 +rank3 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:50:20,729] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:50:21,276] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 16:50:47,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:50:49,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:50:51,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:50:53,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:51:03,188] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 16:51:06,253] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:51:06,743] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 16:51:22,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:51:23,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:51:24,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:51:25,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:51:35,721] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:51:36,911] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 16:51:53,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:51:53,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:51:54,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:51:55,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:52:05,163] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank3 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0815 | Regression_eta: 0.0647 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0693 | LR: 2.20e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3717s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank3 [2026-10-29 17:00:01,012] INFO : Step 38200/50000 rank3 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0774 | Regression_eta: 0.0621 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0628 | LR: 2.18e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8289s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank3 [2026-10-29 17:03:44,995] INFO : Step 38300/50000 rank3 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0948 | Regression_eta: 0.0474 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0602 | LR: 2.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2337s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank3 [2026-10-29 17:07:28,968] INFO : Step 38400/50000 rank3 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0790 | Regression_eta: 0.0417 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0391 | LR: 2.14e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5979s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank3 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank3 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0758 | Regression_eta: 0.0395 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0620 | LR: 2.12e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.5365s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank3 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank3 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2624 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0835 | Regression_eta: 0.0856 | Regression_pt: 0.0880 | Regression_sin_phi: 0.1034 | LR: 2.11e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.4139s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank3 [2026-10-29 17:18:55,030] INFO : Step 38700/50000 rank3 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0703 | Regression_eta: 0.0596 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0640 | LR: 2.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.8739s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank3 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank3 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0722 | Regression_eta: 0.0347 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0568 | LR: 2.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.5805s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank3 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank3 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0775 | Regression_eta: 0.0547 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0748 | LR: 2.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6671s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank3 [2026-10-29 17:30:46,441] INFO : Step 39000/50000 rank3 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0825 | Regression_eta: 0.0571 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0607 | LR: 2.03e-04 | DataLoad Time: 0.0226s | Model Forward Time: 3.6464s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank3 [2026-10-29 17:30:47,443] INFO : Running validation on rank3 +rank3 [2026-10-29 17:30:47,443] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 17:30:47,443] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 17:31:35,653] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:31:36,203] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 17:32:01,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 17:32:02,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 17:32:03,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 17:32:04,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 17:32:14,728] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:32:15,856] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 17:32:31,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 17:32:32,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 17:32:33,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 17:32:34,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 17:32:44,675] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:32:45,863] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 17:33:01,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 17:33:01,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 17:33:02,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 17:33:03,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 17:33:13,244] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 17:37:22,845] INFO : Step 39100/50000 rank3 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0869 | Regression_eta: 0.0450 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0488 | LR: 2.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6379s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank3 [2026-10-29 17:41:18,056] INFO : Step 39200/50000 rank3 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0824 | Regression_eta: 0.0696 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0657 | LR: 2.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7453s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank3 [2026-10-29 17:45:05,747] INFO : Step 39300/50000 rank3 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0802 | Regression_eta: 0.0593 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0749 | LR: 1.98e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9364s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank3 [2026-10-29 17:49:01,211] INFO : Step 39400/50000 rank3 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0706 | Regression_eta: 0.0670 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0666 | LR: 1.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0446s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank3 [2026-10-29 17:53:06,264] INFO : Step 39500/50000 rank3 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0710 | Regression_eta: 0.0466 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0719 | LR: 1.94e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2584s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank3 [2026-10-29 17:56:59,043] INFO : Step 39600/50000 rank3 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0665 | Regression_eta: 0.0371 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0448 | LR: 1.93e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1953s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank3 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank3 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0803 | Regression_eta: 0.0757 | Regression_pt: 0.0837 | Regression_sin_phi: 0.1123 | LR: 1.91e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.2362s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank3 [2026-10-29 18:04:32,041] INFO : Step 39800/50000 rank3 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0705 | Regression_eta: 0.0402 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0561 | LR: 1.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8995s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank3 [2026-10-29 18:08:19,425] INFO : Step 39900/50000 rank3 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0814 | Regression_eta: 0.0624 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0604 | LR: 1.88e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1819s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank3 [2026-10-29 18:12:05,433] INFO : Step 40000/50000 rank3 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0748 | Regression_eta: 0.0531 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0584 | LR: 1.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9401s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank3 [2026-10-29 18:12:06,327] INFO : Running validation on rank3 +rank3 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:12:59,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:12:59,650] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 18:13:23,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:13:24,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:13:25,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:13:26,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:13:36,375] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 18:13:37,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:13:37,721] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 18:13:53,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:13:54,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:13:55,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:13:56,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:14:06,309] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:14:07,581] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 18:14:23,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:14:23,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:14:24,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:14:25,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:14:35,431] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 18:18:48,523] INFO : Step 40100/50000 rank3 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0832 | Regression_eta: 0.0514 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0792 | LR: 1.84e-04 | DataLoad Time: 0.0124s | Model Forward Time: 5.1938s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank3 [2026-10-29 18:22:51,410] INFO : Step 40200/50000 rank3 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0822 | Regression_eta: 0.0520 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0676 | LR: 1.83e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.9281s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank3 [2026-10-29 18:26:53,013] INFO : Step 40300/50000 rank3 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2221 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0893 | Regression_eta: 0.0701 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0713 | LR: 1.81e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4855s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank3 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank3 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0954 | Regression_eta: 0.0725 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0726 | LR: 1.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0126s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank3 [2026-10-29 18:34:31,872] INFO : Step 40500/50000 rank3 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0837 | Regression_eta: 0.0447 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0677 | LR: 1.78e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.4805s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank3 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank3 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2275 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0971 | Regression_eta: 0.0547 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0874 | LR: 1.76e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.1601s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank3 [2026-10-29 18:42:09,670] INFO : Step 40700/50000 rank3 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0824 | Regression_eta: 0.0454 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0660 | LR: 1.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5731s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank3 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank3 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0576 | Regression_energy: 0.1027 | Regression_eta: 0.0458 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0627 | LR: 1.73e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.0688s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank3 [2026-10-29 18:49:49,167] INFO : Step 40900/50000 rank3 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0878 | Regression_eta: 0.0607 | Regression_pt: 0.0921 | Regression_sin_phi: 0.1031 | LR: 1.72e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6409s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank3 [2026-10-29 18:53:41,414] INFO : Step 41000/50000 rank3 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0822 | Regression_eta: 0.0456 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0523 | LR: 1.70e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4671s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank3 [2026-10-29 18:53:42,383] INFO : Running validation on rank3 +rank3 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:54:33,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:54:33,853] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 18:54:59,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:55:00,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:55:01,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:55:02,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:55:12,472] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:55:13,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:55:13,607] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 18:55:29,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:55:30,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:55:31,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:55:32,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:55:42,573] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 18:55:43,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 18:55:59,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:55:59,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:56:00,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:56:01,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:56:11,367] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 19:00:33,894] INFO : Step 41100/50000 rank3 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0822 | Regression_eta: 0.0477 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0812 | LR: 1.69e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.8490s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank3 [2026-10-29 19:04:28,229] INFO : Step 41200/50000 rank3 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0601 | Regression_energy: 0.1141 | Regression_eta: 0.0448 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0576 | LR: 1.67e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9457s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank3 [2026-10-29 19:08:18,072] INFO : Step 41300/50000 rank3 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2513 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0865 | Regression_eta: 0.0691 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0846 | LR: 1.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1400s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank3 [2026-10-29 19:12:02,347] INFO : Step 41400/50000 rank3 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0829 | Regression_eta: 0.0346 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0581 | LR: 1.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9401s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank3 [2026-10-29 19:15:48,709] INFO : Step 41500/50000 rank3 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0798 | Regression_eta: 0.0408 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0755 | LR: 1.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.3269s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank3 [2026-10-29 19:19:48,124] INFO : Step 41600/50000 rank3 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0832 | Regression_eta: 0.0529 | Regression_pt: 0.0871 | Regression_sin_phi: 0.1094 | LR: 1.61e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7651s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank3 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank3 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2527 | Regression_cos_phi: 0.1035 | Regression_energy: 0.0863 | Regression_eta: 0.0638 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0772 | LR: 1.60e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.9482s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank3 [2026-10-29 19:27:38,847] INFO : Step 41800/50000 rank3 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0774 | Regression_eta: 0.0757 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0514 | LR: 1.58e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.3172s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank3 [2026-10-29 19:31:32,457] INFO : Step 41900/50000 rank3 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0799 | Regression_eta: 0.0510 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0798 | LR: 1.57e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.4518s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank3 [2026-10-29 19:35:32,890] INFO : Step 42000/50000 rank3 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0912 | Regression_eta: 0.0487 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0710 | LR: 1.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9405s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank3 [2026-10-29 19:35:33,871] INFO : Running validation on rank3 +rank3 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 19:36:22,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:36:23,272] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 19:36:50,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 19:36:51,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 19:36:52,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 19:36:53,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 19:37:03,072] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:37:04,359] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 19:37:19,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 19:37:20,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 19:37:21,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 19:37:22,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 19:37:32,756] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:37:33,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:37:34,246] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 19:37:49,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 19:37:50,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 19:37:51,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 19:37:51,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 19:38:01,792] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 19:42:30,049] INFO : Step 42100/50000 rank3 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1955 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0917 | Regression_eta: 0.0675 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0940 | LR: 1.54e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0526s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank3 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank3 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0932 | Regression_energy: 0.1083 | Regression_eta: 0.0529 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0985 | LR: 1.53e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8366s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank3 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank3 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 19:50:51,512] INFO : Step 42300/50000 rank3 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2117 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0761 | Regression_eta: 0.0595 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0509 | LR: 1.52e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2020s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank3 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank3 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0742 | Regression_energy: 0.1139 | Regression_eta: 0.0577 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0781 | LR: 1.50e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6761s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank3 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank3 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0931 | Regression_energy: 0.1121 | Regression_eta: 0.0752 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0683 | LR: 1.49e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1953s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank3 [2026-10-29 20:02:59,632] INFO : Step 42600/50000 rank3 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0875 | Regression_eta: 0.0546 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0771 | LR: 1.48e-04 | DataLoad Time: 0.0163s | Model Forward Time: 8.8089s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank3 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank3 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0580 | Regression_energy: 0.1077 | Regression_eta: 0.0503 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0544 | LR: 1.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7991s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank3 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank3 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1719 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0899 | Regression_eta: 0.0384 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0599 | LR: 1.45e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5861s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank3 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank3 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1917 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0676 | Regression_eta: 0.0477 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0536 | LR: 1.44e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.4278s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank3 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank3 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0850 | Regression_eta: 0.0428 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0564 | LR: 1.43e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9433s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank3 [2026-10-29 20:18:48,627] INFO : Running validation on rank3 +rank3 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 20:19:38,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:19:39,386] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 20:20:09,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 20:20:10,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 20:20:11,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 20:20:12,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 20:20:22,604] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:20:23,702] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 20:20:39,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 20:20:40,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 20:20:41,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 20:20:42,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 20:20:52,756] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:20:53,998] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 20:21:09,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 20:21:10,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 20:21:10,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 20:21:11,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 20:21:21,666] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank3 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0916 | Regression_eta: 0.0572 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0817 | LR: 1.42e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.2866s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank3 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank3 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0801 | Regression_eta: 0.0602 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0669 | LR: 1.40e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1800s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank3 [2026-10-29 20:33:23,252] INFO : Step 43300/50000 rank3 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0906 | Regression_eta: 0.0650 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0774 | LR: 1.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3872s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank3 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank3 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0881 | Regression_eta: 0.0468 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0657 | LR: 1.38e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5475s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank3 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank3 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0770 | Regression_energy: 0.1026 | Regression_eta: 0.0452 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0414 | LR: 1.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1827s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank3 [2026-10-29 20:44:45,320] INFO : Step 43600/50000 rank3 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2630 | Regression_cos_phi: 0.1172 | Regression_energy: 0.0815 | Regression_eta: 0.1095 | Regression_pt: 0.0852 | Regression_sin_phi: 0.1161 | LR: 1.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0810s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank3 [2026-10-29 20:48:25,993] INFO : Step 43700/50000 rank3 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0937 | Regression_eta: 0.0446 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0565 | LR: 1.35e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.4594s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank3 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank3 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0783 | Regression_eta: 0.0689 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0955 | LR: 1.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.5357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank3 [2026-10-29 20:56:18,489] INFO : Step 43900/50000 rank3 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2118 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0956 | Regression_eta: 0.0686 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0962 | LR: 1.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3488s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank3 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank3 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0873 | Regression_eta: 0.0551 | Regression_pt: 0.0901 | Regression_sin_phi: 0.1022 | LR: 1.32e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1969s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank3 [2026-10-29 21:00:02,713] INFO : Running validation on rank3 +rank3 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:00:57,488] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:00:58,022] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 21:01:24,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:01:25,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:01:26,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:01:27,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:01:37,649] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:01:38,954] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 21:01:54,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:01:55,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:01:56,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:01:57,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:02:07,360] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:02:08,957] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 21:02:24,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:02:25,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:02:25,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:02:26,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:02:36,592] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 21:06:42,682] INFO : Step 44100/50000 rank3 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0889 | Regression_eta: 0.0628 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0689 | LR: 1.31e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1688s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank3 [2026-10-29 21:10:40,719] INFO : Step 44200/50000 rank3 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0978 | Regression_energy: 0.0954 | Regression_eta: 0.0491 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0650 | LR: 1.30e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9590s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank3 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank3 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0746 | Regression_eta: 0.0520 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0675 | LR: 1.29e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1178s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank3 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank3 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0771 | Regression_eta: 0.0528 | Regression_pt: 0.0801 | Regression_sin_phi: 0.1009 | LR: 1.28e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7692s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank3 [2026-10-29 21:22:16,363] INFO : Step 44500/50000 rank3 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0810 | Regression_eta: 0.0482 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0803 | LR: 1.27e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.8088s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank3 [2026-10-29 21:26:10,594] INFO : Step 44600/50000 rank3 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2051 | Regression_cos_phi: 0.1143 | Regression_energy: 0.0819 | Regression_eta: 0.0576 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0746 | LR: 1.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1480s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank3 [2026-10-29 21:30:16,075] INFO : Step 44700/50000 rank3 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0298 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0948 | Regression_eta: 0.0457 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0780 | LR: 1.25e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.6990s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank3 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank3 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0995 | Regression_energy: 0.0843 | Regression_eta: 0.0508 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0791 | LR: 1.24e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.4087s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank3 [2026-10-29 21:38:11,005] INFO : Step 44900/50000 rank3 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0789 | Regression_eta: 0.0657 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0769 | LR: 1.23e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9525s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank3 [2026-10-29 21:42:13,638] INFO : Step 45000/50000 rank3 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.2166 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0807 | Regression_eta: 0.0418 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0669 | LR: 1.22e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0117s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank3 [2026-10-29 21:42:14,558] INFO : Running validation on rank3 +rank3 [2026-10-29 21:42:14,558] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:42:14,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:43:06,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:43:07,017] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 21:43:32,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:43:33,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:43:34,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:43:36,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:43:46,077] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 21:43:47,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:43:47,738] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 21:44:03,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:44:04,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:44:05,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:44:06,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:44:16,583] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:44:17,710] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 21:44:33,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:44:34,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:44:35,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:44:35,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:44:45,804] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 21:48:55,615] INFO : Step 45100/50000 rank3 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1144 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0850 | Regression_eta: 0.0454 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0570 | LR: 1.21e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.4076s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank3 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank3 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0909 | Regression_eta: 0.0553 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0756 | LR: 1.20e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1529s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank3 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank3 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0831 | Regression_eta: 0.0484 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0698 | LR: 1.19e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0130s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank3 [2026-10-29 22:00:32,927] INFO : Step 45400/50000 rank3 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0741 | Regression_eta: 0.0419 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0812 | LR: 1.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9030s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank3 [2026-10-29 22:04:23,232] INFO : Step 45500/50000 rank3 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0885 | Regression_eta: 0.0592 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0759 | LR: 1.18e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6356s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank3 [2026-10-29 22:08:21,193] INFO : Step 45600/50000 rank3 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0789 | Regression_eta: 0.0452 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0570 | LR: 1.17e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0162s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank3 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank3 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0877 | Regression_eta: 0.0658 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0618 | LR: 1.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0853s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank3 [2026-10-29 22:16:08,602] INFO : Step 45800/50000 rank3 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0762 | Regression_eta: 0.0566 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0735 | LR: 1.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6679s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank3 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank3 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0916 | Regression_eta: 0.0545 | Regression_pt: 0.0981 | Regression_sin_phi: 0.1182 | LR: 1.15e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9611s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank3 [2026-10-29 22:24:24,449] INFO : Step 46000/50000 rank3 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0979 | Regression_eta: 0.0532 | Regression_pt: 0.1029 | Regression_sin_phi: 0.0592 | LR: 1.14e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.2981s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank3 [2026-10-29 22:24:25,403] INFO : Running validation on rank3 +rank3 [2026-10-29 22:24:25,403] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 22:24:25,403] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 22:25:16,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:25:17,231] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 22:25:46,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 22:25:47,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 22:25:48,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 22:25:49,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 22:25:59,197] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 22:25:59,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:26:00,493] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 22:26:16,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 22:26:17,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 22:26:18,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 22:26:20,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 22:26:30,231] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:26:31,921] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 22:26:47,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 22:26:48,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 22:26:49,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 22:26:50,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 22:27:00,017] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 22:31:16,603] INFO : Step 46100/50000 rank3 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0761 | Regression_eta: 0.0512 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0625 | LR: 1.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0432s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank3 [2026-10-29 22:35:12,809] INFO : Step 46200/50000 rank3 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0873 | Regression_eta: 0.0680 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0867 | LR: 1.13e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.9308s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank3 [2026-10-29 22:39:25,150] INFO : Step 46300/50000 rank3 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0850 | Regression_eta: 0.0414 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0654 | LR: 1.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9850s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank3 [2026-10-29 22:43:23,784] INFO : Step 46400/50000 rank3 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0297 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0891 | Regression_eta: 0.0365 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0557 | LR: 1.11e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6673s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank3 [2026-10-29 22:47:17,350] INFO : Step 46500/50000 rank3 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0863 | Regression_eta: 0.0483 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0594 | LR: 1.11e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5753s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank3 [2026-10-29 22:51:20,582] INFO : Step 46600/50000 rank3 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0766 | Regression_eta: 0.0362 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0543 | LR: 1.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.3964s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank3 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank3 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0847 | Regression_eta: 0.0479 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0578 | LR: 1.10e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0467s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank3 [2026-10-29 22:59:26,794] INFO : Step 46800/50000 rank3 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0703 | Regression_eta: 0.0434 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0521 | LR: 1.09e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.9097s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank3 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank3 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0821 | Regression_eta: 0.0409 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0545 | LR: 1.09e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0397s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank3 [2026-10-29 23:08:15,438] INFO : Step 47000/50000 rank3 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0811 | Regression_eta: 0.0428 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0652 | LR: 1.08e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2708s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank3 [2026-10-29 23:08:16,506] INFO : Running validation on rank3 +rank3 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 23:09:07,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:09:08,184] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 23:09:36,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 23:09:38,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 23:09:40,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 23:09:42,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 23:09:52,478] INFO : Time taken to make predictions on device 3 is: 0.31 min +rank3 [2026-10-29 23:09:55,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:09:55,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 23:10:11,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 23:10:12,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 23:10:13,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 23:10:14,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 23:10:24,049] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:10:25,937] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 23:10:41,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 23:10:42,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 23:10:42,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 23:10:43,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 23:10:53,656] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank3 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1730 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0794 | Regression_eta: 0.0568 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0756 | LR: 1.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7055s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank3 [2026-10-29 23:19:19,780] INFO : Step 47200/50000 rank3 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0947 | Regression_energy: 0.0757 | Regression_eta: 0.0599 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0899 | LR: 1.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.6707s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank3 [2026-10-29 23:23:31,492] INFO : Step 47300/50000 rank3 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0756 | Regression_eta: 0.0517 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0587 | LR: 1.06e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.7158s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank3 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank3 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0712 | Regression_eta: 0.0427 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0547 | LR: 1.06e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0192s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank3 [2026-10-29 23:31:38,582] INFO : Step 47500/50000 rank3 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0752 | Regression_eta: 0.0499 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0671 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8368s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank3 [2026-10-29 23:35:34,354] INFO : Step 47600/50000 rank3 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0808 | Regression_eta: 0.0690 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0715 | LR: 1.05e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5028s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank3 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank3 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0724 | Regression_eta: 0.0714 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0756 | LR: 1.05e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.8408s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank3 [2026-10-29 23:43:53,970] INFO : Step 47800/50000 rank3 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0764 | Regression_eta: 0.0427 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0815 | LR: 1.04e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6147s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank3 [2026-10-29 23:47:48,793] INFO : Step 47900/50000 rank3 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0805 | Regression_eta: 0.0894 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0694 | LR: 1.04e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1956s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank3 [2026-10-29 23:51:38,726] INFO : Step 48000/50000 rank3 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0948 | Regression_eta: 0.0507 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0658 | LR: 1.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank3 [2026-10-29 23:51:39,727] INFO : Running validation on rank3 +rank3 [2026-10-29 23:51:39,727] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 23:51:39,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 23:52:31,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:52:31,694] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 23:52:58,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 23:53:00,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 23:53:01,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 23:53:02,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 23:53:12,772] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 23:53:13,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 23:53:29,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 23:53:30,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 23:53:31,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 23:53:32,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 23:53:42,139] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:53:43,871] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 23:53:59,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 23:54:00,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 23:54:00,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 23:54:01,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 23:54:11,659] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 00:00:53,531] INFO : Step 48100/50000 rank3 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0793 | Regression_eta: 0.0500 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0652 | LR: 1.03e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5285s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank3 [2026-10-30 00:04:40,583] INFO : Step 48200/50000 rank3 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0792 | Regression_eta: 0.0505 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0532 | LR: 1.03e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0269s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank3 [2026-10-30 00:08:31,629] INFO : Step 48300/50000 rank3 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0753 | Regression_eta: 0.0504 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0683 | LR: 1.03e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9446s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank3 [2026-10-30 00:12:19,575] INFO : Step 48400/50000 rank3 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1678 | Regression_cos_phi: 0.1127 | Regression_energy: 0.0773 | Regression_eta: 0.0610 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0786 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3347s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank3 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank3 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0683 | Regression_eta: 0.0621 | Regression_pt: 0.0711 | Regression_sin_phi: 0.1016 | LR: 1.02e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.5869s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank3 [2026-10-30 00:19:50,481] INFO : Step 48600/50000 rank3 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2218 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0767 | Regression_eta: 0.0585 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0616 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9691s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank3 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank3 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0721 | Regression_eta: 0.0531 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0584 | LR: 1.02e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0914s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank3 [2026-10-30 00:27:30,340] INFO : Step 48800/50000 rank3 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0712 | Regression_eta: 0.0455 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0547 | LR: 1.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4233s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank3 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank3 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0794 | Regression_eta: 0.0706 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0823 | LR: 1.01e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6442s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank3 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank3 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2484 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0764 | Regression_eta: 0.0724 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0878 | LR: 1.01e-04 | DataLoad Time: 0.0134s | Model Forward Time: 5.6066s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank3 [2026-10-30 00:35:15,046] INFO : Running validation on rank3 +rank3 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:36:03,498] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 00:36:29,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:36:31,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:36:32,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:36:33,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:36:43,413] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-30 00:36:48,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:36:48,746] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 00:37:04,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:37:05,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:37:06,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:37:07,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:37:17,792] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:37:18,485] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 00:37:33,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 00:37:34,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 00:37:35,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 00:37:35,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 00:37:45,856] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 00:41:49,930] INFO : Step 49100/50000 rank3 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0862 | Regression_eta: 0.0530 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0638 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6182s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank3 [2026-10-30 00:45:59,115] INFO : Step 49200/50000 rank3 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0631 | Regression_eta: 0.0590 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0626 | LR: 1.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5915s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank3 [2026-10-30 00:49:54,419] INFO : Step 49300/50000 rank3 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0602 | Regression_energy: 0.1024 | Regression_eta: 0.0576 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0609 | LR: 1.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9851s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank3 [2026-10-30 00:53:44,253] INFO : Step 49400/50000 rank3 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0790 | Regression_eta: 0.0493 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0739 | LR: 1.00e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.9629s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank3 [2026-10-30 00:57:39,408] INFO : Step 49500/50000 rank3 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0666 | Regression_eta: 0.0585 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0551 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0104s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank3 [2026-10-30 01:01:39,148] INFO : Step 49600/50000 rank3 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0717 | Regression_eta: 0.0565 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0698 | LR: 1.00e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8016s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank3 [2026-10-30 01:05:30,841] INFO : Step 49700/50000 rank3 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2164 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0755 | Regression_eta: 0.0612 | Regression_pt: 0.0797 | Regression_sin_phi: 0.1017 | LR: 1.00e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4589s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank3 [2026-10-30 01:09:21,933] INFO : Step 49800/50000 rank3 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0650 | Regression_eta: 0.0454 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0927 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3595s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank3 [2026-10-30 01:13:13,492] INFO : Step 49900/50000 rank3 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0785 | Regression_eta: 0.0560 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0582 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0832s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank3 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank3 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2200 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0792 | Regression_eta: 0.0543 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0939 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8267s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank3 [2026-10-30 01:17:09,618] INFO : Running validation on rank3 +rank3 [2026-10-30 01:17:09,618] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 01:18:02,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:18:03,251] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 01:18:29,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 01:18:30,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 01:18:31,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 01:18:32,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 01:18:42,011] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:18:42,590] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:18:43,092] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 01:18:59,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 01:19:00,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 01:19:01,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 01:19:02,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 01:19:12,010] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 01:19:28,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:19:29,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:19:30,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:19:31,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:19:41,060] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 01:19:41,596] INFO : Training completed. Total time on device 3: 629.366min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.4 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..131c21d1472ab8f5bee89390a8f9f5c85437f513 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.4 @@ -0,0 +1,2483 @@ +rank4 [2026-10-28 11:12:33,139] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-28 11:12:33,140] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank4 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-28 11:12:57,452] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-28 11:12:57,452] INFO : nn0 parameters: 171776 +rank4 [2026-10-28 11:12:57,452] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank4 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,456] INFO : Initializing backbone layer 1 +rank4 [2026-10-28 11:12:57,456] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 2 +rank4 [2026-10-28 11:12:57,459] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,463] INFO : Initializing backbone layer 3 +rank4 [2026-10-28 11:12:57,463] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,466] INFO : Initializing backbone layer 4 +rank4 [2026-10-28 11:12:57,466] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,469] INFO : Initializing backbone layer 5 +rank4 [2026-10-28 11:12:57,469] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:57,472] INFO : Backbone initialization took 0.02s +rank4 [2026-10-28 11:12:57,472] INFO : backbone parameters: 2374656 +rank4 [2026-10-28 11:12:57,472] INFO : Initializing output DNNs +rank4 [2026-10-28 11:12:57,522] INFO : Output DNNs initialization took 0.05s +rank4 [2026-10-28 11:12:57,522] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-28 11:12:57,522] INFO : jagged_attention enabled=True +rank4 [2026-10-28 11:12:57,522] INFO : flash_attn_varlen enabled=True +rank4 [2026-10-28 11:12:57,522] INFO : task_queries enabled=False +rank4 [2026-10-28 11:12:57,522] INFO : classification_norm parameters: 0 +rank4 [2026-10-28 11:12:57,522] INFO : regression_norm parameters: 0 +rank4 [2026-10-28 11:12:57,522] INFO : classification_query parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank4 [2026-10-28 11:12:57,523] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-28 11:12:57,523] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_pid parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_pt parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_eta parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-28 11:12:57,523] INFO : nn_energy parameters: 0 +rank4 [2026-10-28 11:12:57,524] INFO : Total MLPF parameters: 7260244 +rank4 [2026-10-28 11:12:57,524] INFO : MLPF __init__ done +rank4 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank4 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=4 +rank4 [2026-10-28 11:12:57,545] INFO : Moved model to device rank=4 +rank4 [2026-10-28 11:12:57,547] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-28 11:12:57,548] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-28 11:12:57,691] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-28 11:12:57,812] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank4 [2026-10-28 11:12:57,812] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-28 11:12:57,812] INFO : Trainable parameters: 7260244 +rank4 [2026-10-28 11:12:57,813] INFO : Non-trainable parameters: 0 +rank4 [2026-10-28 11:12:57,813] INFO : Total parameters: 7260244 +rank4 [2026-10-28 11:12:57,815] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-28 11:12:57,815] INFO : Getting dataloaders +rank4 [2026-10-28 11:12:57,815] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:03,725] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:04,763] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,002] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-28 11:13:06,002] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-28 11:13:06,002] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,271] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,555] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,820] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-28 11:13:06,820] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-28 11:13:20,895] INFO : Got dataloaders +rank4 [2026-10-28 11:13:20,895] INFO : loader split=train rank=4 len=21108 +rank4 [2026-10-28 11:13:20,895] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-28 11:13:20,895] INFO : Starting training from step 1 to 50000 on rank 4 of 8 +rank4 [2026-10-28 11:13:20,896] INFO : Creating train_iterator +rank4 [2026-10-28 11:13:20,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-28 11:13:35,681] INFO : Created train_iterator +rank4 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank4 [2026-10-28 11:17:20,492] INFO : Step 100/50000 rank4 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7646 | Classification_binary: 0.5357 | Regression_cos_phi: 0.0090 | Regression_energy: 1.2205 | Regression_eta: 0.0235 | Regression_pt: 1.1656 | Regression_sin_phi: 0.0090 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 3.1332s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank4 [2026-10-28 11:20:57,690] INFO : Step 200/50000 rank4 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4566 | Classification_binary: 0.5734 | Regression_cos_phi: 0.4418 | Regression_energy: 0.3623 | Regression_eta: 0.4603 | Regression_pt: 0.3911 | Regression_sin_phi: 0.4368 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.5602s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank4 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank4 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3122 | Classification_binary: 0.4119 | Regression_cos_phi: 0.2537 | Regression_energy: 0.3002 | Regression_eta: 0.2561 | Regression_pt: 0.2965 | Regression_sin_phi: 0.2179 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.7568s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank4 [2026-10-28 11:28:09,552] INFO : Step 400/50000 rank4 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3137 | Classification_binary: 0.4622 | Regression_cos_phi: 0.2941 | Regression_energy: 0.2834 | Regression_eta: 0.3268 | Regression_pt: 0.2933 | Regression_sin_phi: 0.3385 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 3.5812s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank4 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank4 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2927 | Classification_binary: 0.5365 | Regression_cos_phi: 0.3503 | Regression_energy: 0.3121 | Regression_eta: 0.2855 | Regression_pt: 0.3287 | Regression_sin_phi: 0.3469 | LR: 1.00e-03 | DataLoad Time: 0.0148s | Model Forward Time: 4.0279s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank4 [2026-10-28 11:35:32,723] INFO : Step 600/50000 rank4 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2666 | Classification_binary: 0.4965 | Regression_cos_phi: 0.3132 | Regression_energy: 0.2953 | Regression_eta: 0.3279 | Regression_pt: 0.3211 | Regression_sin_phi: 0.3418 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 4.1158s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank4 [2026-10-28 11:39:09,492] INFO : Step 700/50000 rank4 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2233 | Classification_binary: 0.3942 | Regression_cos_phi: 0.2421 | Regression_energy: 0.2758 | Regression_eta: 0.2521 | Regression_pt: 0.2856 | Regression_sin_phi: 0.2420 | LR: 1.00e-03 | DataLoad Time: 0.0176s | Model Forward Time: 3.3985s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank4 [2026-10-28 11:42:49,058] INFO : Step 800/50000 rank4 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2401 | Classification_binary: 0.4912 | Regression_cos_phi: 0.2559 | Regression_energy: 0.2829 | Regression_eta: 0.2885 | Regression_pt: 0.3017 | Regression_sin_phi: 0.2120 | LR: 9.99e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.6503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank4 [2026-10-28 11:46:31,921] INFO : Step 900/50000 rank4 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2788 | Classification_binary: 0.5088 | Regression_cos_phi: 0.3234 | Regression_energy: 0.3111 | Regression_eta: 0.3615 | Regression_pt: 0.3263 | Regression_sin_phi: 0.4558 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5678s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank4 [2026-10-28 11:50:09,951] INFO : Step 1000/50000 rank4 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2215 | Classification_binary: 0.3873 | Regression_cos_phi: 0.1808 | Regression_energy: 0.2322 | Regression_eta: 0.1946 | Regression_pt: 0.2387 | Regression_sin_phi: 0.2206 | LR: 9.99e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.4178s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank4 [2026-10-28 11:50:10,911] INFO : Running validation on rank4 +rank4 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 11:50:58,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:50:59,085] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 11:51:19,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 11:51:20,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 11:51:21,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 11:51:22,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 11:51:32,366] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 11:51:32,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:51:33,312] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 11:51:49,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 11:51:50,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 11:51:51,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 11:51:52,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 11:52:02,053] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 11:52:02,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:52:03,469] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 11:52:18,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 11:52:19,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 11:52:20,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 11:52:21,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 11:52:31,052] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 11:56:43,807] INFO : Step 1100/50000 rank4 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2754 | Classification_binary: 0.4796 | Regression_cos_phi: 0.3505 | Regression_energy: 0.2983 | Regression_eta: 0.3181 | Regression_pt: 0.3028 | Regression_sin_phi: 0.3190 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9232s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank4 [2026-10-28 12:00:30,892] INFO : Step 1200/50000 rank4 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2359 | Classification_binary: 0.5059 | Regression_cos_phi: 0.3863 | Regression_energy: 0.2852 | Regression_eta: 0.2521 | Regression_pt: 0.3003 | Regression_sin_phi: 0.3283 | LR: 9.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3993s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank4 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank4 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1917 | Classification_binary: 0.4010 | Regression_cos_phi: 0.2416 | Regression_energy: 0.2392 | Regression_eta: 0.1787 | Regression_pt: 0.2518 | Regression_sin_phi: 0.1969 | LR: 9.98e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1263s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank4 [2026-10-28 12:08:10,853] INFO : Step 1400/50000 rank4 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2269 | Classification_binary: 0.4394 | Regression_cos_phi: 0.2136 | Regression_energy: 0.2641 | Regression_eta: 0.2065 | Regression_pt: 0.2724 | Regression_sin_phi: 0.2619 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7557s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank4 [2026-10-28 12:11:57,638] INFO : Step 1500/50000 rank4 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1907 | Classification_binary: 0.4648 | Regression_cos_phi: 0.2302 | Regression_energy: 0.2696 | Regression_eta: 0.1900 | Regression_pt: 0.2866 | Regression_sin_phi: 0.2369 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5803s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank4 [2026-10-28 12:15:43,592] INFO : Step 1600/50000 rank4 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1912 | Classification_binary: 0.4632 | Regression_cos_phi: 0.2249 | Regression_energy: 0.2593 | Regression_eta: 0.1977 | Regression_pt: 0.2710 | Regression_sin_phi: 0.2029 | LR: 9.98e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.0378s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank4 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank4 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1784 | Classification_binary: 0.3581 | Regression_cos_phi: 0.2043 | Regression_energy: 0.2641 | Regression_eta: 0.2698 | Regression_pt: 0.2725 | Regression_sin_phi: 0.1963 | LR: 9.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.3921s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank4 [2026-10-28 12:23:17,462] INFO : Step 1800/50000 rank4 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2191 | Classification_binary: 0.4170 | Regression_cos_phi: 0.2420 | Regression_energy: 0.2841 | Regression_eta: 0.1840 | Regression_pt: 0.2977 | Regression_sin_phi: 0.2467 | LR: 9.97e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9425s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank4 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank4 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2536 | Classification_binary: 0.4986 | Regression_cos_phi: 0.3078 | Regression_energy: 0.2518 | Regression_eta: 0.2730 | Regression_pt: 0.2647 | Regression_sin_phi: 0.3176 | LR: 9.97e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5708s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank4 [2026-10-28 12:30:59,046] INFO : Step 2000/50000 rank4 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1691 | Classification_binary: 0.3883 | Regression_cos_phi: 0.2214 | Regression_energy: 0.2151 | Regression_eta: 0.1704 | Regression_pt: 0.2267 | Regression_sin_phi: 0.1678 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1425s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank4 [2026-10-28 12:31:00,064] INFO : Running validation on rank4 +rank4 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 12:31:47,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:31:48,296] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 12:32:13,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:32:14,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:32:15,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:32:16,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:32:26,553] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:32:27,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 12:32:43,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:32:44,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:32:45,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:32:46,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:32:56,495] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:32:57,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:32:57,644] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 12:33:13,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:33:14,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:33:15,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:33:15,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:33:25,802] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 12:37:32,606] INFO : Step 2100/50000 rank4 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2107 | Classification_binary: 0.4025 | Regression_cos_phi: 0.2312 | Regression_energy: 0.2633 | Regression_eta: 0.2052 | Regression_pt: 0.2807 | Regression_sin_phi: 0.2733 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7740s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank4 [2026-10-28 12:41:23,029] INFO : Step 2200/50000 rank4 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2308 | Classification_binary: 0.4875 | Regression_cos_phi: 0.2717 | Regression_energy: 0.2512 | Regression_eta: 0.2860 | Regression_pt: 0.2653 | Regression_sin_phi: 0.2707 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7066s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank4 [2026-10-28 12:45:14,506] INFO : Step 2300/50000 rank4 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1645 | Classification_binary: 0.3904 | Regression_cos_phi: 0.1951 | Regression_energy: 0.2266 | Regression_eta: 0.1668 | Regression_pt: 0.2369 | Regression_sin_phi: 0.2113 | LR: 9.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9865s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank4 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank4 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1622 | Classification_binary: 0.3771 | Regression_cos_phi: 0.2421 | Regression_energy: 0.2056 | Regression_eta: 0.2155 | Regression_pt: 0.2153 | Regression_sin_phi: 0.1741 | LR: 9.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.4492s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank4 [2026-10-28 12:52:58,132] INFO : Step 2500/50000 rank4 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2129 | Classification_binary: 0.4355 | Regression_cos_phi: 0.2142 | Regression_energy: 0.2597 | Regression_eta: 0.1820 | Regression_pt: 0.2735 | Regression_sin_phi: 0.2627 | LR: 9.94e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.3986s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank4 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank4 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1813 | Classification_binary: 0.4615 | Regression_cos_phi: 0.2077 | Regression_energy: 0.2499 | Regression_eta: 0.2094 | Regression_pt: 0.2618 | Regression_sin_phi: 0.2616 | LR: 9.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank4 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank4 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1579 | Classification_binary: 0.3402 | Regression_cos_phi: 0.1696 | Regression_energy: 0.2441 | Regression_eta: 0.1844 | Regression_pt: 0.2596 | Regression_sin_phi: 0.2077 | LR: 9.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0890s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank4 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank4 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1863 | Classification_binary: 0.3901 | Regression_cos_phi: 0.2352 | Regression_energy: 0.2579 | Regression_eta: 0.1979 | Regression_pt: 0.2697 | Regression_sin_phi: 0.2621 | LR: 9.93e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1948s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank4 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank4 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1950 | Classification_binary: 0.4023 | Regression_cos_phi: 0.2403 | Regression_energy: 0.2486 | Regression_eta: 0.2122 | Regression_pt: 0.2625 | Regression_sin_phi: 0.2588 | LR: 9.93e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1934s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank4 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank4 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1501 | Classification_binary: 0.3697 | Regression_cos_phi: 0.1738 | Regression_energy: 0.2113 | Regression_eta: 0.2050 | Regression_pt: 0.2200 | Regression_sin_phi: 0.1990 | LR: 9.92e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5274s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank4 [2026-10-28 13:12:10,495] INFO : Running validation on rank4 +rank4 [2026-10-28 13:12:10,495] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:13:02,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:13:03,432] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 13:13:30,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:13:31,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:13:32,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:13:32,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:13:42,954] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:13:44,105] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 13:14:00,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:14:01,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:14:01,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:14:02,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:14:12,885] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:14:14,292] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 13:14:29,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:14:30,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:14:31,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:14:31,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:14:41,980] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 13:19:02,742] INFO : Step 3100/50000 rank4 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1966 | Classification_binary: 0.3621 | Regression_cos_phi: 0.2590 | Regression_energy: 0.2324 | Regression_eta: 0.1931 | Regression_pt: 0.2460 | Regression_sin_phi: 0.2339 | LR: 9.91e-04 | DataLoad Time: 0.0229s | Model Forward Time: 5.5826s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank4 [2026-10-28 13:23:00,674] INFO : Step 3200/50000 rank4 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2127 | Classification_binary: 0.4624 | Regression_cos_phi: 0.3072 | Regression_energy: 0.2679 | Regression_eta: 0.2707 | Regression_pt: 0.2850 | Regression_sin_phi: 0.3640 | LR: 9.91e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8539s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank4 [2026-10-28 13:26:54,113] INFO : Step 3300/50000 rank4 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1578 | Classification_binary: 0.3930 | Regression_cos_phi: 0.1708 | Regression_energy: 0.2175 | Regression_eta: 0.2085 | Regression_pt: 0.2230 | Regression_sin_phi: 0.1824 | LR: 9.90e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7313s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank4 [2026-10-28 13:30:48,597] INFO : Step 3400/50000 rank4 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1987 | Classification_binary: 0.3251 | Regression_cos_phi: 0.3111 | Regression_energy: 0.2588 | Regression_eta: 0.2267 | Regression_pt: 0.2693 | Regression_sin_phi: 0.2540 | LR: 9.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0757s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank4 [2026-10-28 13:34:47,771] INFO : Step 3500/50000 rank4 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1521 | Classification_binary: 0.3650 | Regression_cos_phi: 0.2070 | Regression_energy: 0.2055 | Regression_eta: 0.1463 | Regression_pt: 0.2116 | Regression_sin_phi: 0.1512 | LR: 9.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.6683s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank4 [2026-10-28 13:38:46,122] INFO : Step 3600/50000 rank4 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1601 | Classification_binary: 0.4170 | Regression_cos_phi: 0.1722 | Regression_energy: 0.2097 | Regression_eta: 0.1593 | Regression_pt: 0.2200 | Regression_sin_phi: 0.2454 | LR: 9.89e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.1032s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank4 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank4 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1414 | Classification_binary: 0.3512 | Regression_cos_phi: 0.1592 | Regression_energy: 0.2057 | Regression_eta: 0.1763 | Regression_pt: 0.2109 | Regression_sin_phi: 0.2038 | LR: 9.88e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.2037s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank4 [2026-10-28 13:46:34,311] INFO : Step 3800/50000 rank4 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.3824 | Regression_cos_phi: 0.2162 | Regression_energy: 0.2718 | Regression_eta: 0.1807 | Regression_pt: 0.2875 | Regression_sin_phi: 0.2201 | LR: 9.87e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9868s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank4 [2026-10-28 13:50:23,157] INFO : Step 3900/50000 rank4 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1888 | Classification_binary: 0.4161 | Regression_cos_phi: 0.2152 | Regression_energy: 0.2319 | Regression_eta: 0.2033 | Regression_pt: 0.2431 | Regression_sin_phi: 0.2582 | LR: 9.87e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2437s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank4 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank4 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1187 | Classification_binary: 0.3418 | Regression_cos_phi: 0.1586 | Regression_energy: 0.2349 | Regression_eta: 0.1269 | Regression_pt: 0.2486 | Regression_sin_phi: 0.1853 | LR: 9.86e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5793s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank4 [2026-10-28 13:54:12,844] INFO : Running validation on rank4 +rank4 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:55:03,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 13:55:33,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:55:34,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:55:35,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:55:36,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:55:46,102] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:55:46,898] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 13:56:02,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:56:04,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:56:05,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:56:05,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:56:15,985] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:56:16,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:56:16,991] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 13:56:32,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:56:33,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:56:33,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:56:34,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:56:44,470] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 14:00:56,375] INFO : Step 4100/50000 rank4 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1624 | Classification_binary: 0.3551 | Regression_cos_phi: 0.1616 | Regression_energy: 0.2494 | Regression_eta: 0.1729 | Regression_pt: 0.2580 | Regression_sin_phi: 0.2231 | LR: 9.85e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9534s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank4 [2026-10-28 14:04:50,729] INFO : Step 4200/50000 rank4 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1732 | Classification_binary: 0.3880 | Regression_cos_phi: 0.2075 | Regression_energy: 0.2299 | Regression_eta: 0.1789 | Regression_pt: 0.2407 | Regression_sin_phi: 0.1699 | LR: 9.84e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4862s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank4 [2026-10-28 14:08:42,269] INFO : Step 4300/50000 rank4 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1896 | Classification_binary: 0.3963 | Regression_cos_phi: 0.2349 | Regression_energy: 0.2238 | Regression_eta: 0.2406 | Regression_pt: 0.2367 | Regression_sin_phi: 0.3127 | LR: 9.84e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1051s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank4 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank4 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1720 | Classification_binary: 0.3078 | Regression_cos_phi: 0.1823 | Regression_energy: 0.1864 | Regression_eta: 0.1307 | Regression_pt: 0.1938 | Regression_sin_phi: 0.1988 | LR: 9.83e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.7766s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank4 [2026-10-28 14:16:26,828] INFO : Step 4500/50000 rank4 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1743 | Classification_binary: 0.4139 | Regression_cos_phi: 0.2071 | Regression_energy: 0.2110 | Regression_eta: 0.1878 | Regression_pt: 0.2236 | Regression_sin_phi: 0.1737 | LR: 9.82e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4674s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank4 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank4 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1553 | Classification_binary: 0.4328 | Regression_cos_phi: 0.2321 | Regression_energy: 0.1815 | Regression_eta: 0.1954 | Regression_pt: 0.1923 | Regression_sin_phi: 0.1827 | LR: 9.81e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1353s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank4 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank4 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.3249 | Regression_cos_phi: 0.1674 | Regression_energy: 0.1857 | Regression_eta: 0.1148 | Regression_pt: 0.1913 | Regression_sin_phi: 0.1442 | LR: 9.81e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3023s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank4 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank4 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1693 | Classification_binary: 0.3590 | Regression_cos_phi: 0.2433 | Regression_energy: 0.1809 | Regression_eta: 0.1670 | Regression_pt: 0.1922 | Regression_sin_phi: 0.2135 | LR: 9.80e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9647s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank4 [2026-10-28 14:32:01,143] INFO : Step 4900/50000 rank4 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1946 | Classification_binary: 0.4798 | Regression_cos_phi: 0.2385 | Regression_energy: 0.2032 | Regression_eta: 0.2225 | Regression_pt: 0.2124 | Regression_sin_phi: 0.2374 | LR: 9.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8446s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank4 [2026-10-28 14:35:52,797] INFO : Step 5000/50000 rank4 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1432 | Classification_binary: 0.3118 | Regression_cos_phi: 0.1894 | Regression_energy: 0.1862 | Regression_eta: 0.1670 | Regression_pt: 0.1922 | Regression_sin_phi: 0.1644 | LR: 9.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8418s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank4 [2026-10-28 14:35:53,776] INFO : Running validation on rank4 +rank4 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 14:36:42,158] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:36:42,707] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 14:37:08,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 14:37:10,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 14:37:11,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 14:37:12,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 14:37:22,041] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:37:22,910] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 14:37:38,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 14:37:39,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 14:37:40,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 14:37:41,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 14:37:51,347] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 14:37:52,214] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:37:52,790] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 14:38:08,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 14:38:09,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 14:38:09,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 14:38:10,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 14:38:20,551] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank4 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1419 | Classification_binary: 0.2802 | Regression_cos_phi: 0.1492 | Regression_energy: 0.2020 | Regression_eta: 0.1246 | Regression_pt: 0.2076 | Regression_sin_phi: 0.1549 | LR: 9.77e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3391s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank4 [2026-10-28 14:46:36,780] INFO : Step 5200/50000 rank4 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1479 | Classification_binary: 0.3432 | Regression_cos_phi: 0.1597 | Regression_energy: 0.2079 | Regression_eta: 0.1352 | Regression_pt: 0.2175 | Regression_sin_phi: 0.1793 | LR: 9.76e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9478s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank4 [2026-10-28 14:50:33,437] INFO : Step 5300/50000 rank4 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.3536 | Regression_cos_phi: 0.1712 | Regression_energy: 0.2040 | Regression_eta: 0.1146 | Regression_pt: 0.2180 | Regression_sin_phi: 0.1776 | LR: 9.75e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1903s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank4 [2026-10-28 14:54:36,491] INFO : Step 5400/50000 rank4 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3399 | Regression_cos_phi: 0.2019 | Regression_energy: 0.2018 | Regression_eta: 0.1658 | Regression_pt: 0.2080 | Regression_sin_phi: 0.2132 | LR: 9.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.4919s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank4 [2026-10-28 14:58:42,458] INFO : Step 5500/50000 rank4 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1878 | Classification_binary: 0.3380 | Regression_cos_phi: 0.2376 | Regression_energy: 0.1926 | Regression_eta: 0.1729 | Regression_pt: 0.2020 | Regression_sin_phi: 0.1498 | LR: 9.73e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5925s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank4 [2026-10-28 15:02:44,094] INFO : Step 5600/50000 rank4 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1697 | Classification_binary: 0.4224 | Regression_cos_phi: 0.2159 | Regression_energy: 0.1959 | Regression_eta: 0.2237 | Regression_pt: 0.2026 | Regression_sin_phi: 0.2394 | LR: 9.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6397s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank4 [2026-10-28 15:06:32,827] INFO : Step 5700/50000 rank4 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.3031 | Regression_cos_phi: 0.1530 | Regression_energy: 0.1869 | Regression_eta: 0.1366 | Regression_pt: 0.1937 | Regression_sin_phi: 0.1639 | LR: 9.71e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.1513s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank4 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank4 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1767 | Classification_binary: 0.3933 | Regression_cos_phi: 0.2319 | Regression_energy: 0.1893 | Regression_eta: 0.2282 | Regression_pt: 0.1981 | Regression_sin_phi: 0.2985 | LR: 9.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5290s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank4 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank4 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1460 | Classification_binary: 0.3924 | Regression_cos_phi: 0.2024 | Regression_energy: 0.2043 | Regression_eta: 0.1963 | Regression_pt: 0.2105 | Regression_sin_phi: 0.2591 | LR: 9.69e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.0723s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank4 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank4 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1771 | Classification_binary: 0.3684 | Regression_cos_phi: 0.2000 | Regression_energy: 0.2053 | Regression_eta: 0.1561 | Regression_pt: 0.2149 | Regression_sin_phi: 0.2021 | LR: 9.68e-04 | DataLoad Time: 0.0125s | Model Forward Time: 6.3982s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank4 [2026-10-28 15:18:46,644] INFO : Running validation on rank4 +rank4 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:19:35,272] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 15:19:56,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 15:19:57,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 15:19:59,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 15:20:00,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 15:20:10,315] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:20:11,685] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 15:20:27,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 15:20:28,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 15:20:29,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 15:20:30,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 15:20:40,482] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:20:40,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:20:41,486] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 15:20:57,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 15:20:57,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 15:20:58,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 15:20:59,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 15:21:09,205] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 15:25:37,243] INFO : Step 6100/50000 rank4 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2141 | Regression_cos_phi: 0.1373 | Regression_energy: 0.2274 | Regression_eta: 0.1102 | Regression_pt: 0.2347 | Regression_sin_phi: 0.1451 | LR: 9.67e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2782s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank4 [2026-10-28 15:29:27,138] INFO : Step 6200/50000 rank4 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1686 | Classification_binary: 0.3565 | Regression_cos_phi: 0.1827 | Regression_energy: 0.2057 | Regression_eta: 0.1858 | Regression_pt: 0.2160 | Regression_sin_phi: 0.2226 | LR: 9.66e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.3975s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank4 [2026-10-28 15:33:16,838] INFO : Step 6300/50000 rank4 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1843 | Classification_binary: 0.3533 | Regression_cos_phi: 0.2357 | Regression_energy: 0.2031 | Regression_eta: 0.1765 | Regression_pt: 0.2132 | Regression_sin_phi: 0.1907 | LR: 9.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2457s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank4 [2026-10-28 15:37:09,470] INFO : Step 6400/50000 rank4 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3088 | Regression_cos_phi: 0.1797 | Regression_energy: 0.1580 | Regression_eta: 0.1492 | Regression_pt: 0.1669 | Regression_sin_phi: 0.1790 | LR: 9.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9974s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank4 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank4 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1471 | Classification_binary: 0.2932 | Regression_cos_phi: 0.1860 | Regression_energy: 0.1683 | Regression_eta: 0.1632 | Regression_pt: 0.1774 | Regression_sin_phi: 0.2202 | LR: 9.63e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6719s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank4 [2026-10-28 15:45:09,041] INFO : Step 6600/50000 rank4 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1861 | Classification_binary: 0.4630 | Regression_cos_phi: 0.2669 | Regression_energy: 0.1786 | Regression_eta: 0.2530 | Regression_pt: 0.1843 | Regression_sin_phi: 0.3171 | LR: 9.62e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2533s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank4 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank4 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1316 | Classification_binary: 0.3223 | Regression_cos_phi: 0.1487 | Regression_energy: 0.1717 | Regression_eta: 0.1172 | Regression_pt: 0.1765 | Regression_sin_phi: 0.1776 | LR: 9.61e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.0311s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank4 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank4 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1522 | Classification_binary: 0.2857 | Regression_cos_phi: 0.1677 | Regression_energy: 0.1718 | Regression_eta: 0.1422 | Regression_pt: 0.1760 | Regression_sin_phi: 0.1973 | LR: 9.60e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8867s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank4 [2026-10-28 15:56:38,730] INFO : Step 6900/50000 rank4 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1563 | Classification_binary: 0.3395 | Regression_cos_phi: 0.1671 | Regression_energy: 0.1771 | Regression_eta: 0.1853 | Regression_pt: 0.1875 | Regression_sin_phi: 0.1811 | LR: 9.58e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.8859s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank4 [2026-10-28 16:00:37,730] INFO : Step 7000/50000 rank4 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.3959 | Regression_cos_phi: 0.1898 | Regression_energy: 0.1868 | Regression_eta: 0.1936 | Regression_pt: 0.1931 | Regression_sin_phi: 0.1955 | LR: 9.57e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4738s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank4 [2026-10-28 16:00:38,672] INFO : Running validation on rank4 +rank4 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:00:38,673] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:01:32,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:01:33,505] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 16:02:00,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:02:01,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:02:02,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:02:03,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:02:13,050] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:02:14,069] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 16:02:30,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:02:31,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:02:32,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:02:33,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:02:43,748] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:02:44,278] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 16:02:59,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:03:00,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:03:01,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:03:01,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:03:11,997] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank4 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1084 | Classification_binary: 0.2733 | Regression_cos_phi: 0.1434 | Regression_energy: 0.1549 | Regression_eta: 0.1164 | Regression_pt: 0.1631 | Regression_sin_phi: 0.2157 | LR: 9.56e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5605s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank4 [2026-10-28 16:11:41,310] INFO : Step 7200/50000 rank4 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3429 | Regression_cos_phi: 0.1977 | Regression_energy: 0.1848 | Regression_eta: 0.1891 | Regression_pt: 0.1965 | Regression_sin_phi: 0.2204 | LR: 9.55e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9560s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank4 [2026-10-28 16:15:45,432] INFO : Step 7300/50000 rank4 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1324 | Classification_binary: 0.3369 | Regression_cos_phi: 0.1919 | Regression_energy: 0.1652 | Regression_eta: 0.1608 | Regression_pt: 0.1674 | Regression_sin_phi: 0.2068 | LR: 9.53e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.9000s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank4 [2026-10-28 16:19:47,791] INFO : Step 7400/50000 rank4 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.3024 | Regression_cos_phi: 0.1899 | Regression_energy: 0.1656 | Regression_eta: 0.1362 | Regression_pt: 0.1676 | Regression_sin_phi: 0.1085 | LR: 9.52e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.6019s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank4 [2026-10-28 16:23:56,337] INFO : Step 7500/50000 rank4 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.2975 | Regression_cos_phi: 0.1372 | Regression_energy: 0.1538 | Regression_eta: 0.1198 | Regression_pt: 0.1598 | Regression_sin_phi: 0.1036 | LR: 9.51e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7919s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank4 [2026-10-28 16:27:58,400] INFO : Step 7600/50000 rank4 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.3698 | Regression_cos_phi: 0.1983 | Regression_energy: 0.2159 | Regression_eta: 0.1642 | Regression_pt: 0.2323 | Regression_sin_phi: 0.2272 | LR: 9.50e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7001s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank4 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank4 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.3190 | Regression_cos_phi: 0.1922 | Regression_energy: 0.1541 | Regression_eta: 0.1411 | Regression_pt: 0.1614 | Regression_sin_phi: 0.1443 | LR: 9.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9799s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank4 [2026-10-28 16:35:55,393] INFO : Step 7800/50000 rank4 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2558 | Regression_cos_phi: 0.1387 | Regression_energy: 0.1730 | Regression_eta: 0.0981 | Regression_pt: 0.1807 | Regression_sin_phi: 0.1039 | LR: 9.47e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2113s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank4 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank4 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1367 | Classification_binary: 0.3334 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1757 | Regression_eta: 0.1200 | Regression_pt: 0.1828 | Regression_sin_phi: 0.1382 | LR: 9.46e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.3425s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank4 [2026-10-28 16:44:12,791] INFO : Step 8000/50000 rank4 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.3487 | Regression_cos_phi: 0.1950 | Regression_energy: 0.1702 | Regression_eta: 0.2036 | Regression_pt: 0.1778 | Regression_sin_phi: 0.2029 | LR: 9.44e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.2431s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank4 [2026-10-28 16:44:13,936] INFO : Running validation on rank4 +rank4 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:45:04,006] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:45:04,563] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 16:45:30,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:45:31,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:45:32,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:45:34,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:45:44,090] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:45:45,474] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 16:46:01,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:46:02,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:46:02,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:46:03,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:46:13,966] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:46:15,124] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 16:46:30,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:46:31,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:46:32,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:46:32,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:46:42,973] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank4 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2775 | Regression_cos_phi: 0.0979 | Regression_energy: 0.1560 | Regression_eta: 0.0919 | Regression_pt: 0.1616 | Regression_sin_phi: 0.1295 | LR: 9.43e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7755s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank4 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank4 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1313 | Classification_binary: 0.3043 | Regression_cos_phi: 0.1821 | Regression_energy: 0.1477 | Regression_eta: 0.1800 | Regression_pt: 0.1534 | Regression_sin_phi: 0.2248 | LR: 9.42e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1558s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank4 [2026-10-28 16:59:31,719] INFO : Step 8300/50000 rank4 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1366 | Classification_binary: 0.3754 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1589 | Regression_eta: 0.1710 | Regression_pt: 0.1653 | Regression_sin_phi: 0.2119 | LR: 9.40e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.9200s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank4 [2026-10-28 17:03:26,315] INFO : Step 8400/50000 rank4 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1064 | Classification_binary: 0.2609 | Regression_cos_phi: 0.1658 | Regression_energy: 0.1531 | Regression_eta: 0.1081 | Regression_pt: 0.1592 | Regression_sin_phi: 0.1637 | LR: 9.39e-04 | DataLoad Time: 0.0229s | Model Forward Time: 6.0444s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank4 [2026-10-28 17:07:18,316] INFO : Step 8500/50000 rank4 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1191 | Classification_binary: 0.2824 | Regression_cos_phi: 0.1643 | Regression_energy: 0.1556 | Regression_eta: 0.1332 | Regression_pt: 0.1605 | Regression_sin_phi: 0.1828 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4969s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank4 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank4 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1258 | Classification_binary: 0.3297 | Regression_cos_phi: 0.1495 | Regression_energy: 0.1644 | Regression_eta: 0.1211 | Regression_pt: 0.1716 | Regression_sin_phi: 0.1605 | LR: 9.36e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1688s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank4 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank4 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1215 | Classification_binary: 0.3191 | Regression_cos_phi: 0.1396 | Regression_energy: 0.1495 | Regression_eta: 0.1281 | Regression_pt: 0.1528 | Regression_sin_phi: 0.1597 | LR: 9.34e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2143s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank4 [2026-10-28 17:19:40,255] INFO : Step 8800/50000 rank4 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2396 | Regression_cos_phi: 0.0828 | Regression_energy: 0.1671 | Regression_eta: 0.1157 | Regression_pt: 0.1729 | Regression_sin_phi: 0.1445 | LR: 9.33e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.4441s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank4 [2026-10-28 17:23:38,393] INFO : Step 8900/50000 rank4 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1393 | Classification_binary: 0.3597 | Regression_cos_phi: 0.2011 | Regression_energy: 0.1576 | Regression_eta: 0.1570 | Regression_pt: 0.1658 | Regression_sin_phi: 0.2120 | LR: 9.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6556s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank4 [2026-10-28 17:27:33,249] INFO : Step 9000/50000 rank4 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1192 | Classification_binary: 0.3604 | Regression_cos_phi: 0.1821 | Regression_energy: 0.1519 | Regression_eta: 0.1372 | Regression_pt: 0.1564 | Regression_sin_phi: 0.1595 | LR: 9.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8308s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank4 [2026-10-28 17:27:34,333] INFO : Running validation on rank4 +rank4 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 17:28:24,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:28:25,265] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 17:28:52,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 17:28:53,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 17:28:54,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 17:28:55,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 17:29:05,073] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:29:06,070] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 17:29:22,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 17:29:23,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 17:29:24,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 17:29:26,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 17:29:36,103] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 17:29:37,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:29:37,691] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 17:29:53,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 17:29:54,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 17:29:54,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 17:29:55,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 17:30:05,497] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 17:34:23,898] INFO : Step 9100/50000 rank4 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1155 | Classification_binary: 0.2802 | Regression_cos_phi: 0.1145 | Regression_energy: 0.1467 | Regression_eta: 0.1128 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1335 | LR: 9.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1043s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank4 [2026-10-28 17:38:24,815] INFO : Step 9200/50000 rank4 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0997 | Classification_binary: 0.2892 | Regression_cos_phi: 0.1482 | Regression_energy: 0.1488 | Regression_eta: 0.1224 | Regression_pt: 0.1581 | Regression_sin_phi: 0.1126 | LR: 9.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3267s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank4 [2026-10-28 17:42:44,642] INFO : Step 9300/50000 rank4 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1208 | Classification_binary: 0.3474 | Regression_cos_phi: 0.1849 | Regression_energy: 0.1550 | Regression_eta: 0.1399 | Regression_pt: 0.1656 | Regression_sin_phi: 0.1469 | LR: 9.25e-04 | DataLoad Time: 0.0243s | Model Forward Time: 4.8256s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank4 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank4 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1130 | Classification_binary: 0.2837 | Regression_cos_phi: 0.1428 | Regression_energy: 0.1364 | Regression_eta: 0.1344 | Regression_pt: 0.1396 | Regression_sin_phi: 0.1748 | LR: 9.24e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4151s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank4 [2026-10-28 17:51:11,694] INFO : Step 9500/50000 rank4 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0919 | Classification_binary: 0.2278 | Regression_cos_phi: 0.1317 | Regression_energy: 0.1487 | Regression_eta: 0.1258 | Regression_pt: 0.1549 | Regression_sin_phi: 0.1340 | LR: 9.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.2527s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank4 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank4 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1298 | Classification_binary: 0.3466 | Regression_cos_phi: 0.2159 | Regression_energy: 0.1411 | Regression_eta: 0.1762 | Regression_pt: 0.1481 | Regression_sin_phi: 0.1639 | LR: 9.21e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.8382s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank4 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank4 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0951 | Classification_binary: 0.3491 | Regression_cos_phi: 0.1401 | Regression_energy: 0.1628 | Regression_eta: 0.1261 | Regression_pt: 0.1721 | Regression_sin_phi: 0.1316 | LR: 9.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9950s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank4 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank4 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1075 | Classification_binary: 0.3005 | Regression_cos_phi: 0.1073 | Regression_energy: 0.1296 | Regression_eta: 0.1060 | Regression_pt: 0.1347 | Regression_sin_phi: 0.1511 | LR: 9.17e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.6896s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank4 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank4 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2487 | Regression_cos_phi: 0.1443 | Regression_energy: 0.1303 | Regression_eta: 0.0918 | Regression_pt: 0.1370 | Regression_sin_phi: 0.1229 | LR: 9.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1764s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank4 [2026-10-28 18:11:35,687] INFO : Step 10000/50000 rank4 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1076 | Classification_binary: 0.3840 | Regression_cos_phi: 0.1513 | Regression_energy: 0.1376 | Regression_eta: 0.1169 | Regression_pt: 0.1455 | Regression_sin_phi: 0.1261 | LR: 9.14e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1281s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank4 [2026-10-28 18:11:36,667] INFO : Running validation on rank4 +rank4 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:12:31,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:12:32,155] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 18:12:58,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:12:59,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:13:00,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:13:01,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:13:11,483] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:13:12,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:13:12,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 18:13:29,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:13:31,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:13:32,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:13:34,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:13:44,561] INFO : Time taken to make predictions on device 4 is: 0.30 min +rank4 [2026-10-28 18:13:48,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:13:49,088] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 18:14:04,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:14:05,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:14:05,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:14:06,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:14:16,390] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 18:18:32,574] INFO : Step 10100/50000 rank4 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1356 | Regression_energy: 0.1453 | Regression_eta: 0.0990 | Regression_pt: 0.1511 | Regression_sin_phi: 0.1635 | LR: 9.12e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.2345s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank4 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank4 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0862 | Classification_binary: 0.2579 | Regression_cos_phi: 0.0995 | Regression_energy: 0.1456 | Regression_eta: 0.1203 | Regression_pt: 0.1523 | Regression_sin_phi: 0.1087 | LR: 9.11e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5686s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank4 [2026-10-28 18:26:52,298] INFO : Step 10300/50000 rank4 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.3196 | Regression_cos_phi: 0.1119 | Regression_energy: 0.1351 | Regression_eta: 0.0913 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0998 | LR: 9.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.3479s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank4 [2026-10-28 18:31:09,158] INFO : Step 10400/50000 rank4 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0898 | Classification_binary: 0.2991 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1237 | Regression_eta: 0.1038 | Regression_pt: 0.1293 | Regression_sin_phi: 0.1172 | LR: 9.07e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9077s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank4 [2026-10-28 18:35:25,945] INFO : Step 10500/50000 rank4 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1036 | Classification_binary: 0.2241 | Regression_cos_phi: 0.1645 | Regression_energy: 0.1115 | Regression_eta: 0.1284 | Regression_pt: 0.1156 | Regression_sin_phi: 0.1433 | LR: 9.06e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.8291s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank4 [2026-10-28 18:39:26,895] INFO : Step 10600/50000 rank4 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.3157 | Regression_cos_phi: 0.1176 | Regression_energy: 0.1337 | Regression_eta: 0.0858 | Regression_pt: 0.1415 | Regression_sin_phi: 0.1274 | LR: 9.04e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.2171s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank4 [2026-10-28 18:43:27,489] INFO : Step 10700/50000 rank4 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.3538 | Regression_cos_phi: 0.1104 | Regression_energy: 0.1292 | Regression_eta: 0.1044 | Regression_pt: 0.1341 | Regression_sin_phi: 0.1236 | LR: 9.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2756s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank4 [2026-10-28 18:47:34,479] INFO : Step 10800/50000 rank4 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1002 | Regression_energy: 0.1324 | Regression_eta: 0.0832 | Regression_pt: 0.1352 | Regression_sin_phi: 0.0827 | LR: 9.00e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.3720s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank4 [2026-10-28 18:51:51,745] INFO : Step 10900/50000 rank4 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2550 | Regression_cos_phi: 0.1216 | Regression_energy: 0.0959 | Regression_eta: 0.0973 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1159 | LR: 8.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.6023s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank4 [2026-10-28 18:55:59,247] INFO : Step 11000/50000 rank4 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.3439 | Regression_cos_phi: 0.1611 | Regression_energy: 0.1238 | Regression_eta: 0.1014 | Regression_pt: 0.1266 | Regression_sin_phi: 0.1476 | LR: 8.97e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.8639s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank4 [2026-10-28 18:56:00,217] INFO : Running validation on rank4 +rank4 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:56:54,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:56:55,374] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 18:57:21,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:57:22,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:57:23,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:57:24,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:57:34,866] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:57:35,272] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:57:35,836] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 18:57:51,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:57:52,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:57:53,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:57:55,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:58:05,117] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:58:06,770] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 18:58:22,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:58:23,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:58:23,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:58:24,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:58:34,396] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 19:03:04,100] INFO : Step 11100/50000 rank4 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.2888 | Regression_cos_phi: 0.1248 | Regression_energy: 0.1176 | Regression_eta: 0.0818 | Regression_pt: 0.1244 | Regression_sin_phi: 0.1093 | LR: 8.95e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.4748s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank4 [2026-10-28 19:07:10,330] INFO : Step 11200/50000 rank4 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0967 | Classification_binary: 0.2539 | Regression_cos_phi: 0.1017 | Regression_energy: 0.1344 | Regression_eta: 0.1526 | Regression_pt: 0.1431 | Regression_sin_phi: 0.1100 | LR: 8.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8089s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank4 [2026-10-28 19:11:04,858] INFO : Step 11300/50000 rank4 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0901 | Classification_binary: 0.2803 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1124 | Regression_eta: 0.0810 | Regression_pt: 0.1157 | Regression_sin_phi: 0.1244 | LR: 8.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2506s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank4 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank4 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.3056 | Regression_cos_phi: 0.1040 | Regression_energy: 0.1319 | Regression_eta: 0.0979 | Regression_pt: 0.1377 | Regression_sin_phi: 0.1147 | LR: 8.89e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0031s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank4 [2026-10-28 19:18:58,942] INFO : Step 11500/50000 rank4 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2239 | Regression_cos_phi: 0.0738 | Regression_energy: 0.1151 | Regression_eta: 0.0822 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0969 | LR: 8.88e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.4533s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank4 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank4 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.2987 | Regression_cos_phi: 0.1522 | Regression_energy: 0.1211 | Regression_eta: 0.1413 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1501 | LR: 8.86e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8314s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank4 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank4 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1126 | Classification_binary: 0.2601 | Regression_cos_phi: 0.0698 | Regression_energy: 0.1241 | Regression_eta: 0.0551 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0774 | LR: 8.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1769s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank4 [2026-10-28 19:30:54,022] INFO : Step 11800/50000 rank4 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.1984 | Regression_cos_phi: 0.1268 | Regression_energy: 0.1177 | Regression_eta: 0.0911 | Regression_pt: 0.1234 | Regression_sin_phi: 0.1380 | LR: 8.82e-04 | DataLoad Time: 0.0236s | Model Forward Time: 6.0650s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank4 [2026-10-28 19:34:55,961] INFO : Step 11900/50000 rank4 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1106 | Classification_binary: 0.2531 | Regression_cos_phi: 0.1168 | Regression_energy: 0.1034 | Regression_eta: 0.1102 | Regression_pt: 0.1063 | Regression_sin_phi: 0.1509 | LR: 8.80e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9526s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank4 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank4 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.3128 | Regression_cos_phi: 0.1270 | Regression_energy: 0.1324 | Regression_eta: 0.0920 | Regression_pt: 0.1357 | Regression_sin_phi: 0.1234 | LR: 8.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3551s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank4 [2026-10-28 19:39:04,976] INFO : Running validation on rank4 +rank4 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 19:39:55,048] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:39:55,582] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 19:40:21,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 19:40:22,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 19:40:23,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 19:40:24,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 19:40:34,373] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:40:34,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:40:35,409] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 19:40:51,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 19:40:52,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 19:40:53,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 19:40:54,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 19:41:04,032] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 19:41:05,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:41:05,609] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 19:41:21,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 19:41:21,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 19:41:22,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 19:41:23,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 19:41:33,203] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 19:45:57,646] INFO : Step 12100/50000 rank4 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.2764 | Regression_cos_phi: 0.1059 | Regression_energy: 0.1186 | Regression_eta: 0.0981 | Regression_pt: 0.1250 | Regression_sin_phi: 0.1129 | LR: 8.76e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8600s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank4 [2026-10-28 19:50:02,390] INFO : Step 12200/50000 rank4 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.2181 | Regression_cos_phi: 0.1355 | Regression_energy: 0.1147 | Regression_eta: 0.1102 | Regression_pt: 0.1169 | Regression_sin_phi: 0.1004 | LR: 8.74e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.9613s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank4 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank4 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.3019 | Regression_cos_phi: 0.1191 | Regression_energy: 0.1127 | Regression_eta: 0.1025 | Regression_pt: 0.1170 | Regression_sin_phi: 0.1218 | LR: 8.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4604s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank4 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank4 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.3173 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1140 | Regression_eta: 0.0925 | Regression_pt: 0.1196 | Regression_sin_phi: 0.1092 | LR: 8.70e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.9466s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank4 [2026-10-28 20:02:34,178] INFO : Step 12500/50000 rank4 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2256 | Regression_cos_phi: 0.1339 | Regression_energy: 0.1080 | Regression_eta: 0.0963 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1198 | LR: 8.68e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.6945s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank4 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank4 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2680 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1103 | Regression_eta: 0.0896 | Regression_pt: 0.1144 | Regression_sin_phi: 0.1092 | LR: 8.66e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.9522s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank4 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank4 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.3000 | Regression_cos_phi: 0.0891 | Regression_energy: 0.1060 | Regression_eta: 0.0708 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0865 | LR: 8.64e-04 | DataLoad Time: 0.0125s | Model Forward Time: 5.2321s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank4 [2026-10-28 20:14:52,957] INFO : Step 12800/50000 rank4 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2432 | Regression_cos_phi: 0.1043 | Regression_energy: 0.1210 | Regression_eta: 0.0854 | Regression_pt: 0.1250 | Regression_sin_phi: 0.1007 | LR: 8.62e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8537s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank4 [2026-10-28 20:19:01,949] INFO : Step 12900/50000 rank4 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0924 | Classification_binary: 0.2446 | Regression_cos_phi: 0.0894 | Regression_energy: 0.1051 | Regression_eta: 0.0887 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1162 | LR: 8.60e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5194s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank4 [2026-10-28 20:23:25,510] INFO : Step 13000/50000 rank4 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0940 | Classification_binary: 0.2903 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0974 | Regression_eta: 0.0883 | Regression_pt: 0.1035 | Regression_sin_phi: 0.1049 | LR: 8.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 10.3374s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank4 [2026-10-28 20:23:26,442] INFO : Running validation on rank4 +rank4 [2026-10-28 20:23:26,442] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 20:23:26,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 20:24:18,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 20:24:45,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 20:24:46,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 20:24:47,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 20:24:48,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 20:24:58,417] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 20:25:04,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:25:04,757] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 20:25:20,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 20:25:21,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 20:25:22,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 20:25:23,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 20:25:33,812] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:25:34,724] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 20:25:50,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 20:25:51,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 20:25:51,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 20:25:52,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 20:26:02,670] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 20:30:28,067] INFO : Step 13100/50000 rank4 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2678 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0988 | Regression_eta: 0.0827 | Regression_pt: 0.1039 | Regression_sin_phi: 0.1178 | LR: 8.56e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.0230s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank4 [2026-10-28 20:34:38,790] INFO : Step 13200/50000 rank4 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1985 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0916 | Regression_eta: 0.0791 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0618 | LR: 8.54e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.0414s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank4 [2026-10-28 20:38:50,933] INFO : Step 13300/50000 rank4 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2287 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0964 | Regression_eta: 0.0977 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0991 | LR: 8.52e-04 | DataLoad Time: 0.0150s | Model Forward Time: 6.4748s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank4 [2026-10-28 20:43:04,348] INFO : Step 13400/50000 rank4 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2887 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0979 | Regression_eta: 0.0826 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0985 | LR: 8.50e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2860s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank4 [2026-10-28 20:47:13,647] INFO : Step 13500/50000 rank4 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0856 | Regression_eta: 0.0607 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0748 | LR: 8.48e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1214s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank4 [2026-10-28 20:51:26,251] INFO : Step 13600/50000 rank4 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0949 | Regression_eta: 0.0807 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0768 | LR: 8.45e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8108s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank4 [2026-10-28 20:55:41,398] INFO : Step 13700/50000 rank4 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2815 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0860 | Regression_eta: 0.0694 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0845 | LR: 8.43e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.8165s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank4 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank4 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2464 | Regression_cos_phi: 0.1043 | Regression_energy: 0.0966 | Regression_eta: 0.0827 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1313 | LR: 8.41e-04 | DataLoad Time: 0.0216s | Model Forward Time: 6.1150s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank4 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank4 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2080 | Regression_cos_phi: 0.0778 | Regression_energy: 0.1012 | Regression_eta: 0.0497 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0770 | LR: 8.39e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.1584s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank4 [2026-10-28 21:08:24,062] INFO : Step 14000/50000 rank4 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0962 | Classification_binary: 0.2676 | Regression_cos_phi: 0.1033 | Regression_energy: 0.1039 | Regression_eta: 0.1077 | Regression_pt: 0.1074 | Regression_sin_phi: 0.1498 | LR: 8.37e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.9210s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank4 [2026-10-28 21:08:24,998] INFO : Running validation on rank4 +rank4 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:09:16,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:09:16,961] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 21:09:43,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:09:44,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:09:45,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:09:46,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:09:56,109] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:09:56,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:09:57,000] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 21:10:13,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:10:14,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:10:15,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:10:16,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:10:26,033] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:10:26,830] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 21:10:43,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:10:43,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:10:44,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:10:46,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:10:56,413] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank4 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2659 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0961 | Regression_eta: 0.0808 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0765 | LR: 8.35e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5884s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank4 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank4 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0771 | Regression_eta: 0.0590 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0781 | LR: 8.32e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.4846s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank4 [2026-10-28 21:24:03,128] INFO : Step 14300/50000 rank4 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2348 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0986 | Regression_eta: 0.0601 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0778 | LR: 8.30e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0232s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank4 [2026-10-28 21:28:09,727] INFO : Step 14400/50000 rank4 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.2767 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0907 | Regression_eta: 0.0695 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0740 | LR: 8.28e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0255s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank4 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank4 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0814 | Regression_eta: 0.0745 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0661 | LR: 8.26e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.3483s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank4 [2026-10-28 21:36:11,052] INFO : Step 14600/50000 rank4 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0976 | Regression_energy: 0.1016 | Regression_eta: 0.0727 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0808 | LR: 8.24e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.9774s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank4 [2026-10-28 21:40:16,608] INFO : Step 14700/50000 rank4 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.3036 | Regression_cos_phi: 0.1278 | Regression_energy: 0.0963 | Regression_eta: 0.0902 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0849 | LR: 8.21e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.1357s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank4 [2026-10-28 21:44:33,910] INFO : Step 14800/50000 rank4 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.2140 | Regression_cos_phi: 0.0743 | Regression_energy: 0.1017 | Regression_eta: 0.0441 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0645 | LR: 8.19e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.3510s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank4 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank4 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0915 | Regression_eta: 0.0762 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0981 | LR: 8.17e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.3227s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank4 [2026-10-28 21:52:53,292] INFO : Step 15000/50000 rank4 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2598 | Regression_cos_phi: 0.0819 | Regression_energy: 0.1096 | Regression_eta: 0.0864 | Regression_pt: 0.1148 | Regression_sin_phi: 0.1022 | LR: 8.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7247s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank4 [2026-10-28 21:52:54,210] INFO : Running validation on rank4 +rank4 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:53:44,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:53:44,569] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 21:54:10,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:54:11,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:54:12,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:54:13,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:54:23,898] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:54:24,123] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:54:24,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 21:54:40,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:54:41,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:54:42,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:54:43,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:54:53,419] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:54:54,913] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 21:55:10,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:55:11,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:55:11,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:55:12,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:55:22,758] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 21:59:40,817] INFO : Step 15100/50000 rank4 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2652 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0919 | Regression_eta: 0.0841 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0751 | LR: 8.12e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.6875s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank4 [2026-10-28 22:03:40,545] INFO : Step 15200/50000 rank4 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1940 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0950 | Regression_eta: 0.1013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.1340 | LR: 8.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9627s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank4 [2026-10-28 22:07:35,577] INFO : Step 15300/50000 rank4 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0947 | Regression_eta: 0.0680 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0850 | LR: 8.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8196s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank4 [2026-10-28 22:11:47,342] INFO : Step 15400/50000 rank4 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2607 | Regression_cos_phi: 0.1172 | Regression_energy: 0.0820 | Regression_eta: 0.0936 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0804 | LR: 8.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank4 [2026-10-28 22:16:06,324] INFO : Step 15500/50000 rank4 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2183 | Regression_cos_phi: 0.1182 | Regression_energy: 0.0717 | Regression_eta: 0.0943 | Regression_pt: 0.0751 | Regression_sin_phi: 0.1271 | LR: 8.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9210s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank4 [2026-10-28 22:20:18,325] INFO : Step 15600/50000 rank4 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0798 | Regression_eta: 0.1140 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1356 | LR: 8.01e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.4195s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank4 [2026-10-28 22:24:30,251] INFO : Step 15700/50000 rank4 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0974 | Classification_binary: 0.2353 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0978 | Regression_eta: 0.0713 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1192 | LR: 7.98e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.9287s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank4 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank4 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2593 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0909 | Regression_eta: 0.0676 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0758 | LR: 7.96e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3020s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank4 [2026-10-28 22:32:57,712] INFO : Step 15900/50000 rank4 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0750 | Regression_eta: 0.0691 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0756 | LR: 7.94e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.3912s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank4 [2026-10-28 22:37:10,742] INFO : Step 16000/50000 rank4 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2577 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0831 | Regression_eta: 0.0865 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0695 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0617s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank4 [2026-10-28 22:37:11,681] INFO : Running validation on rank4 +rank4 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 22:38:03,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:38:04,033] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 22:38:30,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 22:38:31,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 22:38:32,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 22:38:33,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 22:38:43,330] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 22:38:43,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:38:44,323] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 22:39:00,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 22:39:01,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 22:39:03,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 22:39:04,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 22:39:14,250] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 22:39:15,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:39:16,106] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 22:39:31,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 22:39:32,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 22:39:33,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 22:39:34,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 22:39:44,028] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 22:44:03,866] INFO : Step 16100/50000 rank4 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2739 | Regression_cos_phi: 0.1307 | Regression_energy: 0.0887 | Regression_eta: 0.0927 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1111 | LR: 7.89e-04 | DataLoad Time: 0.0123s | Model Forward Time: 4.7827s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank4 [2026-10-28 22:48:09,286] INFO : Step 16200/50000 rank4 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0909 | Regression_eta: 0.0638 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0615 | LR: 7.86e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.6653s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank4 [2026-10-28 22:52:16,253] INFO : Step 16300/50000 rank4 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0912 | Regression_eta: 0.0878 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0778 | LR: 7.84e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.1384s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank4 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank4 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0846 | Regression_eta: 0.0590 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0759 | LR: 7.81e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.4176s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank4 [2026-10-28 23:00:23,725] INFO : Step 16500/50000 rank4 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.2053 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0823 | Regression_eta: 0.0714 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0726 | LR: 7.79e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0001s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank4 [2026-10-28 23:04:26,318] INFO : Step 16600/50000 rank4 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0832 | Regression_eta: 0.0527 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0785 | LR: 7.77e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3640s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank4 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank4 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0771 | Regression_eta: 0.0654 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0878 | LR: 7.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0575s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank4 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank4 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2426 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0771 | Regression_eta: 0.0742 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0747 | LR: 7.72e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.0651s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank4 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank4 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0814 | Regression_eta: 0.0599 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0855 | LR: 7.69e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.7657s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank4 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank4 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0759 | Regression_eta: 0.0589 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0582 | LR: 7.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9350s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank4 [2026-10-28 23:20:37,935] INFO : Running validation on rank4 +rank4 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 23:21:28,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:21:29,099] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 23:21:55,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 23:21:57,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 23:22:00,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 23:22:01,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 23:22:11,764] INFO : Time taken to make predictions on device 4 is: 0.31 min +rank4 [2026-10-28 23:22:12,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:22:12,960] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 23:22:28,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 23:22:29,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 23:22:30,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 23:22:31,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 23:22:41,785] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 23:22:42,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:22:42,878] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 23:22:58,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 23:22:59,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 23:22:59,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 23:23:00,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 23:23:10,519] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 23:27:38,315] INFO : Step 17100/50000 rank4 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0896 | Regression_eta: 0.0557 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0555 | LR: 7.64e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.1681s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank4 [2026-10-28 23:31:38,671] INFO : Step 17200/50000 rank4 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0952 | Regression_eta: 0.0682 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0676 | LR: 7.62e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1955s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank4 [2026-10-28 23:35:49,540] INFO : Step 17300/50000 rank4 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0859 | Regression_eta: 0.0774 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0820 | LR: 7.59e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.5129s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank4 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank4 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2620 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0886 | Regression_eta: 0.0891 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0941 | LR: 7.57e-04 | DataLoad Time: 0.0182s | Model Forward Time: 6.3525s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank4 [2026-10-28 23:43:53,975] INFO : Step 17500/50000 rank4 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0988 | Regression_eta: 0.0637 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0963 | LR: 7.54e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9637s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank4 [2026-10-28 23:48:08,874] INFO : Step 17600/50000 rank4 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0647 | Regression_eta: 0.0415 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0500 | LR: 7.52e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5556s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank4 [2026-10-28 23:52:17,700] INFO : Step 17700/50000 rank4 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0742 | Regression_eta: 0.0699 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0804 | LR: 7.49e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8661s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank4 [2026-10-28 23:56:40,920] INFO : Step 17800/50000 rank4 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0930 | Regression_eta: 0.0742 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0762 | LR: 7.47e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1262s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank4 [2026-10-29 00:00:47,130] INFO : Step 17900/50000 rank4 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0736 | Regression_eta: 0.0616 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0624 | LR: 7.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2723s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank4 [2026-10-29 00:04:53,689] INFO : Step 18000/50000 rank4 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2172 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0915 | Regression_eta: 0.0732 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0867 | LR: 7.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 13.5877s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank4 [2026-10-29 00:04:54,634] INFO : Running validation on rank4 +rank4 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:05:43,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:05:43,821] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 00:06:10,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:06:11,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:06:12,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:06:13,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:06:23,956] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 00:06:24,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 00:06:41,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:06:41,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:06:42,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:06:43,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:06:53,881] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 00:07:10,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:07:11,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:07:12,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:07:12,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:07:22,829] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 00:11:38,772] INFO : Step 18100/50000 rank4 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0854 | Regression_eta: 0.0645 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0642 | LR: 7.39e-04 | DataLoad Time: 0.0273s | Model Forward Time: 4.2896s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank4 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank4 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0856 | Regression_energy: 0.1042 | Regression_eta: 0.1094 | Regression_pt: 0.1070 | Regression_sin_phi: 0.1012 | LR: 7.36e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1143s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank4 [2026-10-29 00:19:54,087] INFO : Step 18300/50000 rank4 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0804 | Regression_eta: 0.0528 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0614 | LR: 7.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8063s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank4 [2026-10-29 00:24:07,390] INFO : Step 18400/50000 rank4 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1004 | Regression_energy: 0.0846 | Regression_eta: 0.0619 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0834 | LR: 7.31e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4810s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank4 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank4 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1984 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0712 | Regression_eta: 0.0655 | Regression_pt: 0.0764 | Regression_sin_phi: 0.1126 | LR: 7.29e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5460s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank4 [2026-10-29 00:31:24,473] INFO : Step 18600/50000 rank4 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2152 | Regression_cos_phi: 0.1249 | Regression_energy: 0.0881 | Regression_eta: 0.1059 | Regression_pt: 0.0933 | Regression_sin_phi: 0.1273 | LR: 7.26e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1902s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank4 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank4 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0779 | Regression_eta: 0.0694 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0910 | LR: 7.24e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.9511s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank4 [2026-10-29 00:38:49,580] INFO : Step 18800/50000 rank4 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0848 | Regression_eta: 0.0422 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0597 | LR: 7.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4360s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank4 [2026-10-29 00:42:28,238] INFO : Step 18900/50000 rank4 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1906 | Regression_cos_phi: 0.1016 | Regression_energy: 0.0600 | Regression_eta: 0.0900 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0834 | LR: 7.18e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0103s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank4 [2026-10-29 00:46:10,110] INFO : Step 19000/50000 rank4 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0921 | Regression_eta: 0.0772 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0968 | LR: 7.16e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1521s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank4 [2026-10-29 00:46:11,054] INFO : Running validation on rank4 +rank4 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:46:58,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:46:59,174] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 00:47:25,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:47:26,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:47:27,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:47:28,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:47:38,053] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 00:47:38,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:47:38,900] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 00:47:54,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:47:55,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:47:56,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:47:57,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:48:07,621] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:48:09,178] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 00:48:24,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:48:25,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:48:26,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:48:26,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:48:36,921] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank4 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0676 | Regression_eta: 0.0503 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0595 | LR: 7.13e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5910s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank4 [2026-10-29 00:56:31,490] INFO : Step 19200/50000 rank4 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0768 | Regression_eta: 0.0649 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0606 | LR: 7.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7719s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank4 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank4 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0827 | Regression_eta: 0.0559 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0898 | LR: 7.08e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6461s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank4 [2026-10-29 01:03:57,849] INFO : Step 19400/50000 rank4 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.2578 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0785 | Regression_eta: 0.0879 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0804 | LR: 7.05e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1742s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank4 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank4 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0772 | Regression_eta: 0.0585 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0930 | LR: 7.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0510s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank4 [2026-10-29 01:11:25,842] INFO : Step 19600/50000 rank4 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0477 | Regression_energy: 0.1034 | Regression_eta: 0.0524 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0568 | LR: 7.00e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9390s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank4 [2026-10-29 01:15:28,511] INFO : Step 19700/50000 rank4 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0800 | Regression_eta: 0.0652 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0839 | LR: 6.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.3553s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank4 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank4 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0712 | Regression_eta: 0.0534 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0748 | LR: 6.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3387s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank4 [2026-10-29 01:23:26,313] INFO : Step 19900/50000 rank4 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0840 | Regression_eta: 0.0936 | Regression_pt: 0.0851 | Regression_sin_phi: 0.1118 | LR: 6.92e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.7468s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank4 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank4 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1895 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0900 | Regression_eta: 0.0805 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0777 | LR: 6.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8757s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank4 [2026-10-29 01:27:16,848] INFO : Running validation on rank4 +rank4 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 01:28:06,612] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:28:07,178] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 01:28:33,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 01:28:35,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 01:28:36,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 01:28:37,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 01:28:47,393] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 01:28:48,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:28:48,587] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 01:29:04,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 01:29:05,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 01:29:06,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 01:29:07,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 01:29:17,413] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 01:29:18,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:29:18,549] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 01:29:33,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 01:29:34,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 01:29:35,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 01:29:35,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 01:29:45,866] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 01:33:54,476] INFO : Step 20100/50000 rank4 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2594 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0964 | Regression_eta: 0.0580 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0839 | LR: 6.86e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.4723s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank4 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank4 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0816 | Regression_eta: 0.0710 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0702 | LR: 6.84e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9229s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank4 [2026-10-29 01:41:55,630] INFO : Step 20300/50000 rank4 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0883 | Regression_eta: 0.0701 | Regression_pt: 0.0947 | Regression_sin_phi: 0.1035 | LR: 6.81e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6949s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank4 [2026-10-29 01:45:46,290] INFO : Step 20400/50000 rank4 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2143 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0671 | Regression_eta: 0.0913 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0830 | LR: 6.78e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1289s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank4 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank4 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.2683 | Regression_cos_phi: 0.1163 | Regression_energy: 0.0824 | Regression_eta: 0.1012 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0926 | LR: 6.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3413s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank4 [2026-10-29 01:53:32,129] INFO : Step 20600/50000 rank4 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0798 | Regression_eta: 0.0639 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0813 | LR: 6.73e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0711s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank4 [2026-10-29 01:57:29,941] INFO : Step 20700/50000 rank4 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2218 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0731 | Regression_eta: 0.0612 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0806 | LR: 6.70e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.6614s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank4 [2026-10-29 02:01:35,426] INFO : Step 20800/50000 rank4 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0994 | Regression_eta: 0.0746 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0915 | LR: 6.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8017s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank4 [2026-10-29 02:05:49,595] INFO : Step 20900/50000 rank4 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.2090 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0701 | Regression_eta: 0.0627 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0869 | LR: 6.65e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.1309s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank4 [2026-10-29 02:09:53,644] INFO : Step 21000/50000 rank4 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0827 | Regression_eta: 0.0751 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0906 | LR: 6.62e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.8432s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank4 [2026-10-29 02:09:54,611] INFO : Running validation on rank4 +rank4 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:10:45,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:10:45,899] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 02:11:12,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:11:13,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:11:14,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:11:15,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:11:25,406] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:11:26,307] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 02:11:43,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:11:44,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:11:45,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:11:47,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:11:57,033] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 02:11:57,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:11:57,803] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 02:12:13,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:12:14,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:12:15,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:12:15,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:12:25,980] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 02:16:43,090] INFO : Step 21100/50000 rank4 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0930 | Regression_eta: 0.0555 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0456 | LR: 6.59e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5460s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank4 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank4 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank4 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0845 | Regression_eta: 0.0489 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0482 | LR: 6.56e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1643s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank4 [2026-10-29 02:24:40,426] INFO : Step 21300/50000 rank4 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2081 | Regression_cos_phi: 0.1397 | Regression_energy: 0.0859 | Regression_eta: 0.0753 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0822 | LR: 6.54e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.1544s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank4 [2026-10-29 02:28:35,147] INFO : Step 21400/50000 rank4 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2781 | Regression_cos_phi: 0.1078 | Regression_energy: 0.0910 | Regression_eta: 0.0901 | Regression_pt: 0.0941 | Regression_sin_phi: 0.1105 | LR: 6.51e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4162s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank4 [2026-10-29 02:32:39,449] INFO : Step 21500/50000 rank4 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0535 | Regression_energy: 0.1018 | Regression_eta: 0.0442 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0607 | LR: 6.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7858s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank4 [2026-10-29 02:36:35,118] INFO : Step 21600/50000 rank4 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1365 | Regression_energy: 0.1088 | Regression_eta: 0.1044 | Regression_pt: 0.1153 | Regression_sin_phi: 0.1185 | LR: 6.45e-04 | DataLoad Time: 0.0231s | Model Forward Time: 3.8427s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank4 [2026-10-29 02:40:30,747] INFO : Step 21700/50000 rank4 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2413 | Regression_cos_phi: 0.0728 | Regression_energy: 0.1001 | Regression_eta: 0.0632 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0808 | LR: 6.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank4 [2026-10-29 02:44:20,672] INFO : Step 21800/50000 rank4 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0861 | Regression_eta: 0.0872 | Regression_pt: 0.0906 | Regression_sin_phi: 0.1017 | LR: 6.40e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.1222s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank4 [2026-10-29 02:48:12,873] INFO : Step 21900/50000 rank4 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0877 | Regression_eta: 0.0808 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0807 | LR: 6.37e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2334s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank4 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank4 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2628 | Regression_cos_phi: 0.1129 | Regression_energy: 0.0831 | Regression_eta: 0.0919 | Regression_pt: 0.0861 | Regression_sin_phi: 0.1142 | LR: 6.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1560s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank4 [2026-10-29 02:52:09,891] INFO : Running validation on rank4 +rank4 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:52:57,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:52:58,579] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 02:53:26,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:53:28,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:53:29,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:53:29,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:53:39,969] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 02:53:56,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:53:57,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:53:58,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:53:59,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:54:09,331] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:54:10,679] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 02:54:25,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:54:26,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:54:27,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:54:27,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:54:37,977] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank4 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2586 | Regression_cos_phi: 0.1065 | Regression_energy: 0.1206 | Regression_eta: 0.0664 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0943 | LR: 6.32e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4351s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank4 [2026-10-29 03:02:36,687] INFO : Step 22200/50000 rank4 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0922 | Regression_eta: 0.0520 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0545 | LR: 6.29e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6563s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank4 [2026-10-29 03:06:28,834] INFO : Step 22300/50000 rank4 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2258 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0951 | Regression_eta: 0.0909 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1091 | LR: 6.26e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8517s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank4 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank4 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2741 | Regression_cos_phi: 0.1055 | Regression_energy: 0.0994 | Regression_eta: 0.0773 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0791 | LR: 6.23e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5934s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank4 [2026-10-29 03:14:16,069] INFO : Step 22500/50000 rank4 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0683 | Regression_energy: 0.1001 | Regression_eta: 0.0544 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1227 | LR: 6.20e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8992s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank4 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank4 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0942 | Regression_eta: 0.0695 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0710 | LR: 6.18e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5794s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank4 [2026-10-29 03:22:04,588] INFO : Step 22700/50000 rank4 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2755 | Regression_cos_phi: 0.1175 | Regression_energy: 0.0934 | Regression_eta: 0.0810 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0778 | LR: 6.15e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3281s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank4 [2026-10-29 03:26:06,870] INFO : Step 22800/50000 rank4 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2558 | Regression_cos_phi: 0.1498 | Regression_energy: 0.0969 | Regression_eta: 0.1416 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0963 | LR: 6.12e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2792s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank4 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank4 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0869 | Regression_eta: 0.0739 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0842 | LR: 6.09e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6100s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank4 [2026-10-29 03:34:03,730] INFO : Step 23000/50000 rank4 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2921 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0962 | Regression_eta: 0.0780 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1097 | LR: 6.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6399s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank4 [2026-10-29 03:34:04,743] INFO : Running validation on rank4 +rank4 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 03:34:52,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:34:53,014] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 03:35:18,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 03:35:19,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 03:35:20,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 03:35:21,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 03:35:31,829] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:35:32,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 03:35:48,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 03:35:49,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 03:35:50,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 03:35:51,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 03:36:01,688] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 03:36:02,378] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:36:02,912] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 03:36:18,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 03:36:19,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 03:36:19,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 03:36:20,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 03:36:30,678] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 03:40:33,164] INFO : Step 23100/50000 rank4 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1061 | Regression_energy: 0.1105 | Regression_eta: 0.0746 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0811 | LR: 6.04e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4168s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank4 [2026-10-29 03:44:14,417] INFO : Step 23200/50000 rank4 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0982 | Regression_eta: 0.0683 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0815 | LR: 6.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4836s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank4 [2026-10-29 03:47:55,309] INFO : Step 23300/50000 rank4 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0972 | Regression_energy: 0.1005 | Regression_eta: 0.1022 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0926 | LR: 5.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9645s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank4 [2026-10-29 03:51:33,798] INFO : Step 23400/50000 rank4 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2847 | Regression_cos_phi: 0.0919 | Regression_energy: 0.1077 | Regression_eta: 0.0760 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1031 | LR: 5.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1986s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank4 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank4 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2493 | Regression_cos_phi: 0.0977 | Regression_energy: 0.1038 | Regression_eta: 0.0891 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1176 | LR: 5.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0145s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank4 [2026-10-29 03:59:15,167] INFO : Step 23600/50000 rank4 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0888 | Regression_eta: 0.0777 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0735 | LR: 5.90e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.9388s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank4 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank4 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2465 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0865 | Regression_eta: 0.0764 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0788 | LR: 5.87e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.3849s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank4 [2026-10-29 04:06:50,323] INFO : Step 23800/50000 rank4 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2546 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0977 | Regression_eta: 0.0725 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0854 | LR: 5.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.0054s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank4 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank4 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0805 | Regression_energy: 0.1121 | Regression_eta: 0.0616 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0782 | LR: 5.81e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3362s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank4 [2026-10-29 04:14:57,721] INFO : Step 24000/50000 rank4 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2697 | Regression_cos_phi: 0.1231 | Regression_energy: 0.1144 | Regression_eta: 0.0988 | Regression_pt: 0.1185 | Regression_sin_phi: 0.1217 | LR: 5.78e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.1454s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank4 [2026-10-29 04:14:58,739] INFO : Running validation on rank4 +rank4 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:15:47,910] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:15:48,538] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 04:16:15,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:16:16,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:16:17,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:16:18,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:16:28,494] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:16:29,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 04:16:45,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:16:46,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:16:47,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:16:49,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:16:59,037] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:16:59,583] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 04:17:15,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:17:15,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:17:16,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:17:17,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:17:27,167] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 04:21:44,038] INFO : Step 24100/50000 rank4 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2741 | Regression_cos_phi: 0.1087 | Regression_energy: 0.1032 | Regression_eta: 0.0958 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0986 | LR: 5.75e-04 | DataLoad Time: 0.0206s | Model Forward Time: 7.7519s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank4 [2026-10-29 04:25:33,907] INFO : Step 24200/50000 rank4 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0902 | Regression_eta: 0.0594 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0932 | LR: 5.73e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2949s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank4 [2026-10-29 04:29:22,270] INFO : Step 24300/50000 rank4 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2124 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1013 | Regression_eta: 0.0658 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0976 | LR: 5.70e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0407s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank4 [2026-10-29 04:33:07,495] INFO : Step 24400/50000 rank4 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.3029 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1234 | Regression_eta: 0.1340 | Regression_pt: 0.1298 | Regression_sin_phi: 0.1496 | LR: 5.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2375s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank4 [2026-10-29 04:36:57,191] INFO : Step 24500/50000 rank4 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0789 | Classification_binary: 0.2321 | Regression_cos_phi: 0.1128 | Regression_energy: 0.1111 | Regression_eta: 0.0815 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0948 | LR: 5.64e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3249s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank4 [2026-10-29 04:40:40,561] INFO : Step 24600/50000 rank4 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0991 | Regression_energy: 0.1161 | Regression_eta: 0.0636 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0827 | LR: 5.61e-04 | DataLoad Time: 0.0265s | Model Forward Time: 3.9882s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank4 [2026-10-29 04:44:29,989] INFO : Step 24700/50000 rank4 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1431 | Regression_energy: 0.1226 | Regression_eta: 0.1051 | Regression_pt: 0.1294 | Regression_sin_phi: 0.1137 | LR: 5.58e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5089s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank4 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank4 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2362 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0930 | Regression_eta: 0.0773 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0980 | LR: 5.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8636s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank4 [2026-10-29 04:52:00,597] INFO : Step 24900/50000 rank4 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0692 | Regression_energy: 0.1029 | Regression_eta: 0.0714 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0966 | LR: 5.53e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9782s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank4 [2026-10-29 04:55:44,334] INFO : Step 25000/50000 rank4 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.2524 | Regression_cos_phi: 0.0955 | Regression_energy: 0.1232 | Regression_eta: 0.0869 | Regression_pt: 0.1289 | Regression_sin_phi: 0.1013 | LR: 5.50e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2502s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank4 [2026-10-29 04:55:45,304] INFO : Running validation on rank4 +rank4 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:55:45,305] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:56:33,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:56:34,566] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 04:57:00,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:57:01,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:57:02,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:57:03,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:57:13,218] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:57:13,409] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:57:13,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 04:57:29,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:57:31,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:57:32,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:57:33,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:57:43,169] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:57:44,417] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 04:57:59,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:58:00,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:58:00,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:58:01,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:58:11,674] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 05:02:15,911] INFO : Step 25100/50000 rank4 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2534 | Regression_cos_phi: 0.1043 | Regression_energy: 0.0846 | Regression_eta: 0.1020 | Regression_pt: 0.0914 | Regression_sin_phi: 0.1119 | LR: 5.47e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7073s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank4 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank4 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0870 | Regression_energy: 0.1077 | Regression_eta: 0.0600 | Regression_pt: 0.1129 | Regression_sin_phi: 0.1055 | LR: 5.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2804s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank4 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank4 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1987 | Regression_cos_phi: 0.1104 | Regression_energy: 0.1077 | Regression_eta: 0.0777 | Regression_pt: 0.1137 | Regression_sin_phi: 0.1287 | LR: 5.42e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.8554s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank4 [2026-10-29 05:13:35,662] INFO : Step 25400/50000 rank4 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0954 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1495 | Regression_energy: 0.1208 | Regression_eta: 0.1474 | Regression_pt: 0.1247 | Regression_sin_phi: 0.1415 | LR: 5.39e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0665s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank4 [2026-10-29 05:17:27,535] INFO : Step 25500/50000 rank4 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2793 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1378 | Regression_eta: 0.1092 | Regression_pt: 0.1437 | Regression_sin_phi: 0.1127 | LR: 5.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0769s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank4 [2026-10-29 05:21:16,672] INFO : Step 25600/50000 rank4 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0969 | Regression_energy: 0.1182 | Regression_eta: 0.1153 | Regression_pt: 0.1223 | Regression_sin_phi: 0.1200 | LR: 5.33e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8751s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank4 [2026-10-29 05:25:00,205] INFO : Step 25700/50000 rank4 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2740 | Regression_cos_phi: 0.0842 | Regression_energy: 0.1341 | Regression_eta: 0.0914 | Regression_pt: 0.1391 | Regression_sin_phi: 0.1197 | LR: 5.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8886s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank4 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank4 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2849 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1208 | Regression_eta: 0.0958 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0892 | LR: 5.27e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5172s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank4 [2026-10-29 05:32:33,908] INFO : Step 25900/50000 rank4 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1029 | Regression_eta: 0.0954 | Regression_pt: 0.1081 | Regression_sin_phi: 0.1108 | LR: 5.25e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8714s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank4 [2026-10-29 05:36:28,636] INFO : Step 26000/50000 rank4 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2190 | Regression_cos_phi: 0.1138 | Regression_energy: 0.1255 | Regression_eta: 0.0877 | Regression_pt: 0.1300 | Regression_sin_phi: 0.1454 | LR: 5.22e-04 | DataLoad Time: 0.0177s | Model Forward Time: 9.2332s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank4 [2026-10-29 05:36:29,553] INFO : Running validation on rank4 +rank4 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 05:37:19,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:37:20,020] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 05:37:45,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 05:37:46,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 05:37:47,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 05:37:48,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 05:37:58,012] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 05:37:58,606] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 05:38:15,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 05:38:16,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 05:38:17,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 05:38:18,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 05:38:28,253] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:38:28,891] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 05:38:44,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 05:38:45,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 05:38:45,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 05:38:46,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 05:38:56,577] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 05:43:04,442] INFO : Step 26100/50000 rank4 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2927 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1203 | Regression_eta: 0.0946 | Regression_pt: 0.1264 | Regression_sin_phi: 0.1246 | LR: 5.19e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8585s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank4 [2026-10-29 05:47:03,509] INFO : Step 26200/50000 rank4 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0993 | Regression_energy: 0.1193 | Regression_eta: 0.0645 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0591 | LR: 5.16e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.2709s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank4 [2026-10-29 05:50:52,485] INFO : Step 26300/50000 rank4 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0921 | Regression_energy: 0.1034 | Regression_eta: 0.0947 | Regression_pt: 0.1076 | Regression_sin_phi: 0.1008 | LR: 5.13e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3831s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank4 [2026-10-29 05:54:52,697] INFO : Step 26400/50000 rank4 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2914 | Regression_cos_phi: 0.1230 | Regression_energy: 0.1770 | Regression_eta: 0.0964 | Regression_pt: 0.1865 | Regression_sin_phi: 0.1450 | LR: 5.10e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3830s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank4 [2026-10-29 05:58:42,121] INFO : Step 26500/50000 rank4 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2467 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1333 | Regression_eta: 0.0822 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0995 | LR: 5.08e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.4049s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank4 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank4 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0901 | Classification_binary: 0.1934 | Regression_cos_phi: 0.1250 | Regression_energy: 0.1038 | Regression_eta: 0.0903 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0928 | LR: 5.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7658s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank4 [2026-10-29 06:06:39,601] INFO : Step 26700/50000 rank4 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.2554 | Regression_cos_phi: 0.1180 | Regression_energy: 0.1385 | Regression_eta: 0.0821 | Regression_pt: 0.1477 | Regression_sin_phi: 0.1431 | LR: 5.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.6719s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank4 [2026-10-29 06:10:41,255] INFO : Step 26800/50000 rank4 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.3033 | Regression_cos_phi: 0.0964 | Regression_energy: 0.1449 | Regression_eta: 0.1033 | Regression_pt: 0.1515 | Regression_sin_phi: 0.1356 | LR: 4.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2409s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank4 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank4 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0692 | Regression_energy: 0.1156 | Regression_eta: 0.0724 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0863 | LR: 4.96e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8905s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank4 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank4 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1123 | Regression_eta: 0.0792 | Regression_pt: 0.1183 | Regression_sin_phi: 0.1306 | LR: 4.94e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.9860s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank4 [2026-10-29 06:18:43,933] INFO : Running validation on rank4 +rank4 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 06:19:32,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:19:33,147] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 06:20:00,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 06:20:01,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 06:20:02,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 06:20:03,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 06:20:13,681] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 06:20:14,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:20:14,781] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 06:20:31,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 06:20:32,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 06:20:33,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 06:20:34,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 06:20:44,646] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:20:46,297] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 06:21:02,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 06:21:02,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 06:21:03,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 06:21:04,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 06:21:14,172] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 06:25:28,750] INFO : Step 27100/50000 rank4 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2645 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1047 | Regression_eta: 0.1138 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0909 | LR: 4.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7011s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank4 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank4 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2425 | Regression_cos_phi: 0.1326 | Regression_energy: 0.1140 | Regression_eta: 0.0836 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0997 | LR: 4.88e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5756s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank4 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank4 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0604 | Regression_energy: 0.1051 | Regression_eta: 0.0684 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0761 | LR: 4.85e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.9957s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank4 [2026-10-29 06:37:36,742] INFO : Step 27400/50000 rank4 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2608 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1223 | Regression_eta: 0.0811 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0947 | LR: 4.82e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5104s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank4 [2026-10-29 06:41:28,998] INFO : Step 27500/50000 rank4 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2632 | Regression_cos_phi: 0.1100 | Regression_energy: 0.1284 | Regression_eta: 0.0898 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0953 | LR: 4.80e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.9233s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank4 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank4 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0942 | Regression_eta: 0.1025 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0790 | LR: 4.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8531s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank4 [2026-10-29 06:49:16,023] INFO : Step 27700/50000 rank4 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.2201 | Regression_cos_phi: 0.0840 | Regression_energy: 0.1191 | Regression_eta: 0.0654 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0839 | LR: 4.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0209s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank4 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank4 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1292 | Regression_energy: 0.1128 | Regression_eta: 0.1031 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0865 | LR: 4.71e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.6684s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank4 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank4 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0616 | Regression_energy: 0.1220 | Regression_eta: 0.0569 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0663 | LR: 4.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5358s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank4 [2026-10-29 07:01:10,309] INFO : Step 28000/50000 rank4 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2184 | Regression_cos_phi: 0.0949 | Regression_energy: 0.1228 | Regression_eta: 0.0851 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0863 | LR: 4.66e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3486s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank4 [2026-10-29 07:01:11,150] INFO : Running validation on rank4 +rank4 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:02:01,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:02:02,349] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 07:02:30,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:02:31,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:02:32,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:02:32,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:02:42,966] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:02:44,046] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 07:03:00,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:03:01,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:03:02,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:03:03,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:03:13,446] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:03:14,474] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 07:03:29,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:03:30,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:03:31,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:03:32,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:03:42,024] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 07:08:04,579] INFO : Step 28100/50000 rank4 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2335 | Regression_cos_phi: 0.1324 | Regression_energy: 0.1157 | Regression_eta: 0.0752 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0998 | LR: 4.63e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1500s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank4 [2026-10-29 07:11:56,590] INFO : Step 28200/50000 rank4 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2549 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1164 | Regression_eta: 0.0802 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0909 | LR: 4.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2162s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank4 [2026-10-29 07:16:10,739] INFO : Step 28300/50000 rank4 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0903 | Regression_energy: 0.1189 | Regression_eta: 0.0680 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0629 | LR: 4.57e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8328s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank4 [2026-10-29 07:20:03,894] INFO : Step 28400/50000 rank4 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1194 | Regression_eta: 0.0531 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0696 | LR: 4.55e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0877s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank4 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank4 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2658 | Regression_cos_phi: 0.1016 | Regression_energy: 0.1227 | Regression_eta: 0.1239 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0876 | LR: 4.52e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1329s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank4 [2026-10-29 07:27:52,535] INFO : Step 28600/50000 rank4 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0700 | Regression_energy: 0.1240 | Regression_eta: 0.0484 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0559 | LR: 4.49e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2890s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank4 [2026-10-29 07:31:49,173] INFO : Step 28700/50000 rank4 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2315 | Regression_cos_phi: 0.0954 | Regression_energy: 0.1005 | Regression_eta: 0.0721 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0870 | LR: 4.46e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0939s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank4 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank4 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2400 | Regression_cos_phi: 0.1266 | Regression_energy: 0.1116 | Regression_eta: 0.0868 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1292 | LR: 4.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4532s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank4 [2026-10-29 07:39:38,718] INFO : Step 28900/50000 rank4 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2188 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0934 | Regression_eta: 0.0758 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0781 | LR: 4.41e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3189s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank4 [2026-10-29 07:43:43,866] INFO : Step 29000/50000 rank4 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0918 | Regression_energy: 0.1284 | Regression_eta: 0.0583 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0938 | LR: 4.38e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9068s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank4 [2026-10-29 07:43:44,819] INFO : Running validation on rank4 +rank4 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:44:33,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:44:33,717] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 07:44:59,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:45:00,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:45:01,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:45:02,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:45:12,418] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:45:12,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:45:13,371] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 07:45:29,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:45:30,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:45:31,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:45:32,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:45:42,546] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:45:43,720] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 07:45:59,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:45:59,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:46:00,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:46:01,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:46:11,105] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 07:50:42,138] INFO : Step 29100/50000 rank4 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1944 | Regression_cos_phi: 0.1067 | Regression_energy: 0.1057 | Regression_eta: 0.0662 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0817 | LR: 4.35e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.7000s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank4 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank4 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2316 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0995 | Regression_eta: 0.0687 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0823 | LR: 4.33e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2672s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank4 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank4 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0762 | Regression_energy: 0.1144 | Regression_eta: 0.0552 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0844 | LR: 4.30e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.9990s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank4 [2026-10-29 08:02:11,406] INFO : Step 29400/50000 rank4 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0975 | Regression_eta: 0.0694 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0668 | LR: 4.27e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5183s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank4 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank4 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2561 | Regression_cos_phi: 0.0798 | Regression_energy: 0.1054 | Regression_eta: 0.0637 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0859 | LR: 4.24e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4989s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank4 [2026-10-29 08:09:49,627] INFO : Step 29600/50000 rank4 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0209 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0576 | Regression_energy: 0.1046 | Regression_eta: 0.0521 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0543 | LR: 4.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7110s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank4 [2026-10-29 08:13:31,666] INFO : Step 29700/50000 rank4 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2045 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0842 | Regression_eta: 0.0596 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0892 | LR: 4.19e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.1846s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank4 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank4 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2360 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1064 | Regression_eta: 0.0693 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0739 | LR: 4.16e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3849s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank4 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank4 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0664 | Regression_energy: 0.1049 | Regression_eta: 0.0510 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0685 | LR: 4.14e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9953s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank4 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank4 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0910 | Regression_eta: 0.0361 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0520 | LR: 4.11e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7058s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank4 [2026-10-29 08:24:57,340] INFO : Running validation on rank4 +rank4 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 08:25:45,090] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:25:45,651] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 08:26:11,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 08:26:12,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 08:26:13,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 08:26:14,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 08:26:24,234] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 08:26:24,835] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:26:25,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 08:26:41,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 08:26:42,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 08:26:43,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 08:26:44,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 08:26:54,053] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:26:55,054] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 08:27:10,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 08:27:11,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 08:27:12,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 08:27:12,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 08:27:22,918] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank4 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0879 | Regression_eta: 0.0780 | Regression_pt: 0.0907 | Regression_sin_phi: 0.1036 | LR: 4.08e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8512s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank4 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank4 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2126 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0960 | Regression_eta: 0.0583 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0858 | LR: 4.06e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5323s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank4 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank4 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0918 | Regression_eta: 0.0833 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0934 | LR: 4.03e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.1784s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank4 [2026-10-29 08:43:19,168] INFO : Step 30400/50000 rank4 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0660 | Regression_energy: 0.1019 | Regression_eta: 0.0413 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0581 | LR: 4.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5065s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank4 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank4 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.2094 | Regression_cos_phi: 0.1109 | Regression_energy: 0.0833 | Regression_eta: 0.0620 | Regression_pt: 0.0867 | Regression_sin_phi: 0.1056 | LR: 3.98e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7626s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank4 [2026-10-29 08:50:50,678] INFO : Step 30600/50000 rank4 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0874 | Regression_eta: 0.0918 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0745 | LR: 3.95e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5469s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank4 [2026-10-29 08:54:35,590] INFO : Step 30700/50000 rank4 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0979 | Regression_eta: 0.0526 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0562 | LR: 3.92e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.2952s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank4 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank4 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2455 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0970 | Regression_eta: 0.0760 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0965 | LR: 3.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2074s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank4 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank4 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0844 | Regression_eta: 0.0594 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0844 | LR: 3.87e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9282s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank4 [2026-10-29 09:05:52,366] INFO : Step 31000/50000 rank4 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0771 | Regression_eta: 0.0486 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0792 | LR: 3.84e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.1879s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank4 [2026-10-29 09:05:53,358] INFO : Running validation on rank4 +rank4 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:06:41,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:06:42,225] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 09:07:08,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:07:09,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:07:10,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:07:11,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:07:21,362] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:07:22,582] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 09:07:38,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:07:39,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:07:40,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:07:41,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:07:51,125] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:07:52,510] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 09:08:08,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:08:09,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:08:09,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:08:10,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:08:20,494] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank4 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2145 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0825 | Regression_eta: 0.0655 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0732 | LR: 3.82e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4612s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank4 [2026-10-29 09:16:29,207] INFO : Step 31200/50000 rank4 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2610 | Regression_cos_phi: 0.1295 | Regression_energy: 0.0779 | Regression_eta: 0.0747 | Regression_pt: 0.0808 | Regression_sin_phi: 0.1118 | LR: 3.79e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5355s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank4 [2026-10-29 09:20:24,830] INFO : Step 31300/50000 rank4 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0859 | Regression_eta: 0.0502 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0665 | LR: 3.76e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.4672s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank4 [2026-10-29 09:24:20,383] INFO : Step 31400/50000 rank4 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1916 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0803 | Regression_eta: 0.0525 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0782 | LR: 3.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8370s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank4 [2026-10-29 09:28:17,180] INFO : Step 31500/50000 rank4 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0802 | Regression_eta: 0.0658 | Regression_pt: 0.0827 | Regression_sin_phi: 0.1115 | LR: 3.71e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.3624s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank4 [2026-10-29 09:32:16,798] INFO : Step 31600/50000 rank4 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0855 | Regression_eta: 0.0483 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0575 | LR: 3.69e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8627s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank4 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank4 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0725 | Regression_eta: 0.0438 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0569 | LR: 3.66e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1357s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank4 [2026-10-29 09:40:22,622] INFO : Step 31800/50000 rank4 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2281 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0810 | Regression_eta: 0.0798 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0974 | LR: 3.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3533s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank4 [2026-10-29 09:48:15,345] INFO : Step 31900/50000 rank4 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.2062 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0721 | Regression_eta: 0.0524 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0600 | LR: 3.61e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8689s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank4 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank4 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0864 | Regression_eta: 0.0693 | Regression_pt: 0.0920 | Regression_sin_phi: 0.0807 | LR: 3.58e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1230s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank4 [2026-10-29 09:52:26,792] INFO : Running validation on rank4 +rank4 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:53:17,424] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:53:17,943] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 09:53:45,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:53:47,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:53:48,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:53:49,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:53:59,705] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:54:01,008] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 09:54:17,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:54:17,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:54:18,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:54:19,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:54:29,846] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:54:30,792] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 09:54:46,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:54:47,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:54:47,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:54:48,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:54:58,711] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank4 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0823 | Regression_eta: 0.0576 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0741 | LR: 3.56e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.9467s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank4 [2026-10-29 10:03:26,459] INFO : Step 32200/50000 rank4 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0880 | Regression_eta: 0.0522 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0768 | LR: 3.53e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5230s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank4 [2026-10-29 10:07:22,792] INFO : Step 32300/50000 rank4 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0740 | Regression_eta: 0.0414 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0713 | LR: 3.51e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5877s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank4 [2026-10-29 10:11:42,985] INFO : Step 32400/50000 rank4 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0975 | Regression_eta: 0.0534 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0640 | LR: 3.48e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0252s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank4 [2026-10-29 10:15:56,366] INFO : Step 32500/50000 rank4 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0789 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0806 | Regression_eta: 0.0794 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0904 | LR: 3.46e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5250s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank4 [2026-10-29 10:20:13,268] INFO : Step 32600/50000 rank4 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0813 | Regression_eta: 0.0450 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0811 | LR: 3.43e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4756s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank4 [2026-10-29 10:24:11,750] INFO : Step 32700/50000 rank4 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0950 | Regression_eta: 0.0556 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0825 | LR: 3.41e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0510s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank4 [2026-10-29 10:28:05,810] INFO : Step 32800/50000 rank4 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0839 | Regression_eta: 0.0572 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0694 | LR: 3.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5520s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank4 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank4 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0780 | Regression_eta: 0.0532 | Regression_pt: 0.0818 | Regression_sin_phi: 0.1052 | LR: 3.36e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.8074s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank4 [2026-10-29 10:36:14,934] INFO : Step 33000/50000 rank4 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0959 | Regression_eta: 0.0514 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0605 | LR: 3.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7659s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank4 [2026-10-29 10:36:15,962] INFO : Running validation on rank4 +rank4 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 10:37:03,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:37:04,370] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 10:37:32,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 10:37:34,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 10:37:36,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 10:37:37,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 10:37:47,821] INFO : Time taken to make predictions on device 4 is: 0.30 min +rank4 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:37:48,815] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 10:38:05,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 10:38:06,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 10:38:07,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 10:38:08,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 10:38:18,083] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:38:21,705] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 10:38:37,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 10:38:38,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 10:38:39,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 10:38:39,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 10:38:49,800] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 10:43:12,059] INFO : Step 33100/50000 rank4 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1731 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0832 | Regression_eta: 0.0449 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0686 | LR: 3.31e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.9489s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank4 [2026-10-29 10:47:22,843] INFO : Step 33200/50000 rank4 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0773 | Regression_eta: 0.0729 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0787 | LR: 3.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6969s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank4 [2026-10-29 10:51:33,085] INFO : Step 33300/50000 rank4 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0825 | Regression_eta: 0.0747 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0828 | LR: 3.26e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4511s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank4 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank4 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0741 | Regression_eta: 0.0568 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0747 | LR: 3.23e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5452s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank4 [2026-10-29 10:59:48,623] INFO : Step 33500/50000 rank4 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2428 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0811 | Regression_eta: 0.0757 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0841 | LR: 3.21e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.9164s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank4 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank4 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2308 | Regression_cos_phi: 0.1001 | Regression_energy: 0.0791 | Regression_eta: 0.0768 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0940 | LR: 3.19e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2005s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank4 [2026-10-29 11:09:08,678] INFO : Step 33700/50000 rank4 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0888 | Regression_eta: 0.0579 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0514 | LR: 3.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3160s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank4 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank4 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0700 | Regression_eta: 0.0499 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0649 | LR: 3.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3677s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank4 [2026-10-29 11:16:52,541] INFO : Step 33900/50000 rank4 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.2195 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0826 | Regression_eta: 0.0555 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0990 | LR: 3.11e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank4 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank4 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0715 | Regression_eta: 0.0532 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0619 | LR: 3.09e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9487s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank4 [2026-10-29 11:20:48,852] INFO : Running validation on rank4 +rank4 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 11:21:42,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:21:42,910] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 11:22:09,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 11:22:10,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 11:22:11,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 11:22:12,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 11:22:22,045] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:22:22,977] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 11:22:39,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 11:22:40,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 11:22:41,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 11:22:42,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 11:22:52,113] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:22:53,478] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 11:23:08,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 11:23:09,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 11:23:10,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 11:23:11,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 11:23:21,042] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 11:27:42,516] INFO : Step 34100/50000 rank4 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0809 | Regression_eta: 0.0392 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0502 | LR: 3.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 7.5172s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank4 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank4 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0761 | Regression_eta: 0.0487 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0696 | LR: 3.04e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7881s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank4 [2026-10-29 11:35:56,836] INFO : Step 34300/50000 rank4 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0735 | Regression_eta: 0.0526 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0685 | LR: 3.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.2638s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank4 [2026-10-29 11:40:08,369] INFO : Step 34400/50000 rank4 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0795 | Regression_eta: 0.0686 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0805 | LR: 2.99e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.9075s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank4 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank4 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2499 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0814 | Regression_eta: 0.0739 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1166 | LR: 2.97e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.5065s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank4 [2026-10-29 11:47:58,875] INFO : Step 34600/50000 rank4 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0879 | Regression_eta: 0.0464 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0813 | LR: 2.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6059s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank4 [2026-10-29 11:52:00,068] INFO : Step 34700/50000 rank4 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0852 | Regression_eta: 0.0618 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0562 | LR: 2.92e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.5530s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank4 [2026-10-29 11:55:53,802] INFO : Step 34800/50000 rank4 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0828 | Regression_eta: 0.0779 | Regression_pt: 0.0864 | Regression_sin_phi: 0.1222 | LR: 2.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8590s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank4 [2026-10-29 11:59:58,708] INFO : Step 34900/50000 rank4 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0883 | Regression_eta: 0.0420 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0654 | LR: 2.88e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1698s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank4 [2026-10-29 12:04:06,317] INFO : Step 35000/50000 rank4 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2188 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0893 | Regression_eta: 0.0610 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0857 | LR: 2.85e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4587s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank4 [2026-10-29 12:04:07,348] INFO : Running validation on rank4 +rank4 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 12:05:04,842] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:05:05,387] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 12:05:31,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 12:05:32,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 12:05:33,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 12:05:34,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 12:05:44,841] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 12:06:02,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 12:06:02,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 12:06:03,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 12:06:04,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 12:06:14,841] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 12:06:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:06:16,078] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 12:06:31,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 12:06:32,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 12:06:33,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 12:06:34,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 12:06:44,030] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank4 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0819 | Regression_eta: 0.0543 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0738 | LR: 2.83e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.6536s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank4 [2026-10-29 12:15:06,116] INFO : Step 35200/50000 rank4 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0744 | Regression_eta: 0.0653 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0806 | LR: 2.81e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0589s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank4 [2026-10-29 12:18:57,447] INFO : Step 35300/50000 rank4 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0880 | Regression_eta: 0.0683 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0612 | LR: 2.79e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.8162s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank4 [2026-10-29 12:23:00,202] INFO : Step 35400/50000 rank4 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0740 | Regression_eta: 0.0559 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0518 | LR: 2.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3956s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank4 [2026-10-29 12:27:14,475] INFO : Step 35500/50000 rank4 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0701 | Regression_eta: 0.0498 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0641 | LR: 2.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2684s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank4 [2026-10-29 12:31:17,345] INFO : Step 35600/50000 rank4 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0896 | Regression_eta: 0.0811 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0872 | LR: 2.72e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1605s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank4 [2026-10-29 12:35:31,257] INFO : Step 35700/50000 rank4 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0897 | Regression_eta: 0.0485 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0667 | LR: 2.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2787s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank4 [2026-10-29 14:49:26,319] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-29 14:49:26,337] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank4 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-29 14:49:53,253] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank4 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank4 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank4 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank4 [2026-10-29 14:49:53,279] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,282] INFO : Initializing backbone layer 3 +rank4 [2026-10-29 14:49:53,282] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,285] INFO : Initializing backbone layer 4 +rank4 [2026-10-29 14:49:53,286] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,289] INFO : Initializing backbone layer 5 +rank4 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank4 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank4 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank4 [2026-10-29 14:49:53,341] INFO : Output DNNs initialization took 0.05s +rank4 [2026-10-29 14:49:53,341] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-29 14:49:53,342] INFO : jagged_attention enabled=True +rank4 [2026-10-29 14:49:53,342] INFO : flash_attn_varlen enabled=True +rank4 [2026-10-29 14:49:53,342] INFO : task_queries enabled=False +rank4 [2026-10-29 14:49:53,342] INFO : classification_norm parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : regression_norm parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : classification_query parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank4 [2026-10-29 14:49:53,342] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-29 14:49:53,342] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : nn_pid parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : nn_pt parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : nn_eta parameters: 0 +rank4 [2026-10-29 14:49:53,342] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-29 14:49:53,343] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-29 14:49:53,343] INFO : nn_energy parameters: 0 +rank4 [2026-10-29 14:49:53,343] INFO : Total MLPF parameters: 7260244 +rank4 [2026-10-29 14:49:53,343] INFO : MLPF __init__ done +rank4 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank4 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=4 +rank4 [2026-10-29 14:49:53,361] INFO : Moved model to device rank=4 +rank4 [2026-10-29 14:49:53,362] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-29 14:49:53,683] INFO : Restoring training from step 35001 +rank4 [2026-10-29 14:49:53,689] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank4 [2026-10-29 14:49:53,689] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-29 14:49:53,909] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-29 14:49:54,026] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank4 [2026-10-29 14:49:54,026] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-29 14:49:54,027] INFO : Trainable parameters: 7260244 +rank4 [2026-10-29 14:49:54,027] INFO : Non-trainable parameters: 0 +rank4 [2026-10-29 14:49:54,027] INFO : Total parameters: 7260244 +rank4 [2026-10-29 14:49:54,030] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-29 14:49:54,030] INFO : Getting dataloaders +rank4 [2026-10-29 14:49:54,030] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:01,382] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:02,342] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:03,556] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-29 14:50:03,557] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-29 14:50:03,557] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:03,819] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:04,069] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:04,413] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-29 14:50:04,413] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-29 14:50:18,046] INFO : Got dataloaders +rank4 [2026-10-29 14:50:18,046] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank4 [2026-10-29 14:50:18,046] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank4 [2026-10-29 14:50:18,046] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank4 [2026-10-29 14:50:18,046] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank4 [2026-10-29 14:50:19,597] INFO : loader split=train rank=4 len=21108 +rank4 [2026-10-29 14:50:19,597] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-29 14:50:19,597] INFO : Starting training from step 35001 to 50000 on rank 4 of 8 +rank4 [2026-10-29 14:50:19,597] INFO : Creating train_iterator +rank4 [2026-10-29 14:50:19,597] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank4 [2026-10-29 14:50:33,403] INFO : Created train_iterator +rank4 [2026-10-29 14:54:16,364] INFO : Step 35100/50000 rank4 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0835 | Regression_eta: 0.0454 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0801 | LR: 2.83e-04 | DataLoad Time: 0.0168s | Model Forward Time: 2.8682s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank4 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank4 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0818 | Regression_eta: 0.0622 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0661 | LR: 2.81e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.5106s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank4 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank4 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0893 | Regression_eta: 0.0607 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0564 | LR: 2.79e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5908s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank4 [2026-10-29 15:05:10,669] INFO : Step 35400/50000 rank4 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0601 | Regression_eta: 0.0387 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0480 | LR: 2.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.4260s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank4 [2026-10-29 15:08:49,315] INFO : Step 35500/50000 rank4 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0780 | Regression_eta: 0.0459 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0600 | LR: 2.74e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.4533s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank4 [2026-10-29 15:12:27,418] INFO : Step 35600/50000 rank4 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2324 | Regression_cos_phi: 0.1177 | Regression_energy: 0.0841 | Regression_eta: 0.0602 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0801 | LR: 2.72e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6868s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank4 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank4 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0798 | Regression_eta: 0.0479 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0720 | LR: 2.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.3528s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank4 [2026-10-29 15:19:47,615] INFO : Step 35800/50000 rank4 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0746 | Regression_eta: 0.0482 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0666 | LR: 2.68e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.4871s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank4 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank4 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2501 | Regression_cos_phi: 0.0724 | Regression_energy: 0.1041 | Regression_eta: 0.0578 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0777 | LR: 2.65e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8623s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank4 [2026-10-29 15:27:19,182] INFO : Step 36000/50000 rank4 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0643 | Regression_eta: 0.0605 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0674 | LR: 2.63e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8293s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank4 [2026-10-29 15:27:20,114] INFO : Running validation on rank4 +rank4 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 15:28:08,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:28:08,876] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 15:28:36,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 15:28:37,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 15:28:38,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 15:28:39,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 15:28:49,417] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 15:28:49,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:28:50,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 15:29:06,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 15:29:07,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 15:29:08,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 15:29:08,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 15:29:18,992] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:29:20,196] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 15:29:35,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 15:29:36,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 15:29:37,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 15:29:37,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 15:29:47,984] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank4 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0894 | Regression_eta: 0.0470 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0673 | LR: 2.61e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.9079s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank4 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank4 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0752 | Regression_eta: 0.0619 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0831 | LR: 2.59e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5675s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank4 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank4 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0822 | Regression_eta: 0.0485 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0445 | LR: 2.57e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5110s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank4 [2026-10-29 15:45:12,807] INFO : Step 36400/50000 rank4 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0868 | Regression_eta: 0.0495 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0994 | LR: 2.55e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.2449s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank4 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank4 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0873 | Regression_eta: 0.0453 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0586 | LR: 2.52e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0162s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank4 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank4 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2413 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0816 | Regression_eta: 0.0716 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0757 | LR: 2.50e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7037s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank4 [2026-10-29 15:56:51,301] INFO : Step 36700/50000 rank4 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0804 | Regression_eta: 0.0796 | Regression_pt: 0.0838 | Regression_sin_phi: 0.1139 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7861s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank4 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank4 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0862 | Regression_eta: 0.0405 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0488 | LR: 2.46e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.4189s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank4 [2026-10-29 16:04:39,809] INFO : Step 36900/50000 rank4 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2425 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0787 | Regression_eta: 0.0624 | Regression_pt: 0.0824 | Regression_sin_phi: 0.1086 | LR: 2.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7923s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank4 [2026-10-29 16:08:30,144] INFO : Step 37000/50000 rank4 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0838 | Regression_eta: 0.0561 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0557 | LR: 2.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9049s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank4 [2026-10-29 16:08:31,022] INFO : Running validation on rank4 +rank4 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:09:20,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:09:20,760] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 16:09:46,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:09:47,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:09:48,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:09:49,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:09:59,240] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 16:10:15,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:10:16,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:10:17,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:10:18,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:10:28,934] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:10:30,579] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 16:10:46,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:10:46,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:10:47,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:10:48,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:10:58,180] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 16:15:06,403] INFO : Step 37100/50000 rank4 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1675 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0832 | Regression_eta: 0.0888 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0724 | LR: 2.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8498s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank4 [2026-10-29 16:18:46,282] INFO : Step 37200/50000 rank4 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2121 | Regression_cos_phi: 0.1120 | Regression_energy: 0.0777 | Regression_eta: 0.0631 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0811 | LR: 2.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2297s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank4 [2026-10-29 16:22:16,729] INFO : Step 37300/50000 rank4 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.2248 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0758 | Regression_eta: 0.0611 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0983 | LR: 2.36e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5814s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank4 [2026-10-29 16:26:01,823] INFO : Step 37400/50000 rank4 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0739 | Regression_eta: 0.0427 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0372 | LR: 2.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9410s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank4 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank4 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0950 | Regression_eta: 0.0673 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0449 | LR: 2.32e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3475s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank4 [2026-10-29 16:34:14,940] INFO : Step 37600/50000 rank4 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2464 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0880 | Regression_eta: 0.0615 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0798 | LR: 2.30e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1119s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank4 [2026-10-29 16:38:00,808] INFO : Step 37700/50000 rank4 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0870 | Regression_eta: 0.0478 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0661 | LR: 2.28e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1663s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank4 [2026-10-29 16:41:51,410] INFO : Step 37800/50000 rank4 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1488 | Regression_cos_phi: 0.1013 | Regression_energy: 0.0906 | Regression_eta: 0.0437 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0455 | LR: 2.26e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.2933s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank4 [2026-10-29 16:45:39,811] INFO : Step 37900/50000 rank4 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0821 | Regression_eta: 0.0515 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0698 | LR: 2.24e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8073s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank4 [2026-10-29 16:49:32,191] INFO : Step 38000/50000 rank4 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0697 | Regression_eta: 0.0550 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0731 | LR: 2.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8893s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank4 [2026-10-29 16:49:33,136] INFO : Running validation on rank4 +rank4 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:50:21,279] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 16:50:48,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:50:50,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:50:52,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:50:55,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:51:05,398] INFO : Time taken to make predictions on device 4 is: 0.32 min +rank4 [2026-10-29 16:51:06,261] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:51:06,743] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 16:51:22,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:51:23,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:51:24,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:51:25,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:51:35,744] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:51:36,911] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 16:51:52,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:51:53,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:51:54,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:51:55,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:52:05,032] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank4 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0693 | Regression_eta: 0.0451 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0589 | LR: 2.20e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3710s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank4 [2026-10-29 17:00:01,012] INFO : Step 38200/50000 rank4 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0800 | Regression_eta: 0.0486 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0663 | LR: 2.18e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8286s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank4 [2026-10-29 17:03:44,996] INFO : Step 38300/50000 rank4 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.2049 | Regression_cos_phi: 0.1135 | Regression_energy: 0.0771 | Regression_eta: 0.0530 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0820 | LR: 2.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2336s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank4 [2026-10-29 17:07:28,967] INFO : Step 38400/50000 rank4 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0756 | Regression_eta: 0.0447 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0519 | LR: 2.14e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.5967s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank4 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank4 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0705 | Regression_eta: 0.0551 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0444 | LR: 2.12e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.5370s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank4 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank4 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0777 | Regression_eta: 0.0502 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0778 | LR: 2.11e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4122s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank4 [2026-10-29 17:18:55,030] INFO : Step 38700/50000 rank4 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0701 | Regression_eta: 0.0536 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0689 | LR: 2.09e-04 | DataLoad Time: 0.0120s | Model Forward Time: 3.8815s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank4 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank4 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0807 | Regression_eta: 0.0663 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0638 | LR: 2.07e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.5806s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank4 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank4 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0748 | Regression_eta: 0.0561 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0783 | LR: 2.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6669s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank4 [2026-10-29 17:30:46,440] INFO : Step 39000/50000 rank4 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0736 | Regression_eta: 0.0514 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0849 | LR: 2.03e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.6482s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank4 [2026-10-29 17:30:47,444] INFO : Running validation on rank4 +rank4 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 17:31:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:31:36,203] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 17:32:01,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 17:32:02,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 17:32:04,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 17:32:04,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 17:32:14,954] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:32:15,856] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 17:32:31,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 17:32:32,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 17:32:33,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 17:32:34,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 17:32:44,637] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:32:45,924] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 17:33:01,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 17:33:02,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 17:33:02,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 17:33:03,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 17:33:13,461] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank4 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1723 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0863 | Regression_eta: 0.0550 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0752 | LR: 2.01e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.6378s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank4 [2026-10-29 17:41:18,056] INFO : Step 39200/50000 rank4 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0758 | Regression_eta: 0.0676 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0683 | LR: 2.00e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7453s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank4 [2026-10-29 17:45:05,746] INFO : Step 39300/50000 rank4 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.2200 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0818 | Regression_eta: 0.0753 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0799 | LR: 1.98e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9361s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank4 [2026-10-29 17:49:01,212] INFO : Step 39400/50000 rank4 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0760 | Regression_eta: 0.0571 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0630 | LR: 1.96e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0454s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank4 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank4 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0706 | Regression_eta: 0.0396 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0501 | LR: 1.94e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2575s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank4 [2026-10-29 17:56:59,044] INFO : Step 39600/50000 rank4 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0869 | Regression_eta: 0.0503 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0755 | LR: 1.93e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1954s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank4 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank4 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0774 | Regression_eta: 0.0469 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0591 | LR: 1.91e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2351s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank4 [2026-10-29 18:04:32,042] INFO : Step 39800/50000 rank4 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0706 | Regression_eta: 0.0417 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0576 | LR: 1.89e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9013s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank4 [2026-10-29 18:08:19,424] INFO : Step 39900/50000 rank4 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0783 | Regression_eta: 0.0540 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0837 | LR: 1.88e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.1835s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank4 [2026-10-29 18:12:05,434] INFO : Step 40000/50000 rank4 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0915 | Regression_eta: 0.0417 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0407 | LR: 1.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9412s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank4 [2026-10-29 18:12:06,327] INFO : Running validation on rank4 +rank4 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:12:06,327] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:12:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:12:59,647] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 18:13:23,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:13:24,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:13:25,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:13:26,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:13:36,424] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:13:37,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:13:37,720] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 18:13:53,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:13:54,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:13:55,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:13:56,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:14:06,648] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:14:07,478] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 18:14:22,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:14:23,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:14:24,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:14:25,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:14:35,130] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 18:18:48,523] INFO : Step 40100/50000 rank4 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0626 | Regression_eta: 0.0485 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0710 | LR: 1.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1853s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank4 [2026-10-29 18:22:51,411] INFO : Step 40200/50000 rank4 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0899 | Regression_eta: 0.0437 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0565 | LR: 1.83e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9284s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank4 [2026-10-29 18:26:53,014] INFO : Step 40300/50000 rank4 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0825 | Regression_eta: 0.0509 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0864 | LR: 1.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4855s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank4 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank4 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0789 | Regression_eta: 0.0575 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0610 | LR: 1.79e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0117s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank4 [2026-10-29 18:34:31,871] INFO : Step 40500/50000 rank4 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0878 | Regression_eta: 0.0335 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0464 | LR: 1.78e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.4811s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank4 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank4 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0908 | Regression_eta: 0.0548 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0693 | LR: 1.76e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1596s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank4 [2026-10-29 18:42:09,669] INFO : Step 40700/50000 rank4 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0892 | Regression_eta: 0.0456 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0695 | LR: 1.75e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5723s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank4 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank4 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1419 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0668 | Regression_eta: 0.0525 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0588 | LR: 1.73e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.0688s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank4 [2026-10-29 18:49:49,168] INFO : Step 40900/50000 rank4 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0865 | Regression_eta: 0.0637 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0704 | LR: 1.72e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.6423s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank4 [2026-10-29 18:53:41,413] INFO : Step 41000/50000 rank4 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0920 | Regression_eta: 0.0517 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0891 | LR: 1.70e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4676s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank4 [2026-10-29 18:53:42,383] INFO : Running validation on rank4 +rank4 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:53:42,383] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:54:33,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:54:33,853] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 18:54:59,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:55:00,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:55:01,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:55:02,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:55:12,647] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:55:13,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:55:13,610] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 18:55:29,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:55:30,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:55:31,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:55:32,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:55:42,666] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 18:55:59,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:56:00,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:56:00,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:56:01,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:56:11,708] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 19:00:33,893] INFO : Step 41100/50000 rank4 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0671 | Regression_eta: 0.0581 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0714 | LR: 1.69e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.8488s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank4 [2026-10-29 19:04:28,228] INFO : Step 41200/50000 rank4 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0880 | Regression_energy: 0.1011 | Regression_eta: 0.0520 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0640 | LR: 1.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9456s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank4 [2026-10-29 19:08:18,073] INFO : Step 41300/50000 rank4 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2329 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0973 | Regression_eta: 0.0537 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0588 | LR: 1.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1405s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank4 [2026-10-29 19:12:02,347] INFO : Step 41400/50000 rank4 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0887 | Regression_eta: 0.0374 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0582 | LR: 1.64e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9384s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank4 [2026-10-29 19:15:48,707] INFO : Step 41500/50000 rank4 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0872 | Regression_eta: 0.0508 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0559 | LR: 1.63e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.3270s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank4 [2026-10-29 19:19:48,124] INFO : Step 41600/50000 rank4 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2218 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0969 | Regression_eta: 0.0612 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0529 | LR: 1.61e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7656s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank4 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank4 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1969 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0814 | Regression_eta: 0.0767 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0615 | LR: 1.60e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.9478s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank4 [2026-10-29 19:27:38,848] INFO : Step 41800/50000 rank4 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0889 | Regression_eta: 0.0586 | Regression_pt: 0.0928 | Regression_sin_phi: 0.1050 | LR: 1.58e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.3170s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank4 [2026-10-29 19:31:32,456] INFO : Step 41900/50000 rank4 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2188 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0850 | Regression_eta: 0.0578 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0684 | LR: 1.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4509s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank4 [2026-10-29 19:35:32,892] INFO : Step 42000/50000 rank4 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.2297 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0814 | Regression_eta: 0.0558 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1025 | LR: 1.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9422s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank4 [2026-10-29 19:35:33,871] INFO : Running validation on rank4 +rank4 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 19:36:22,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:36:23,272] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 19:36:50,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 19:36:51,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 19:36:52,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 19:36:53,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 19:37:03,462] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:37:04,354] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 19:37:20,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 19:37:21,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 19:37:22,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 19:37:23,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 19:37:33,133] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:37:33,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:37:34,031] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 19:37:49,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 19:37:49,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 19:37:50,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 19:37:51,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 19:38:01,101] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 19:42:30,054] INFO : Step 42100/50000 rank4 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1658 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0899 | Regression_eta: 0.0690 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0754 | LR: 1.54e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0587s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank4 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank4 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0948 | Regression_eta: 0.0517 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0712 | LR: 1.53e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8367s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank4 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank4 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 19:50:51,512] INFO : Step 42300/50000 rank4 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0892 | Regression_eta: 0.0481 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0680 | LR: 1.52e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2015s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank4 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank4 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0844 | Regression_eta: 0.0587 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0506 | LR: 1.50e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6758s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank4 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank4 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0874 | Regression_eta: 0.0586 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0986 | LR: 1.49e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank4 [2026-10-29 20:02:59,631] INFO : Step 42600/50000 rank4 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2274 | Regression_cos_phi: 0.1015 | Regression_energy: 0.1132 | Regression_eta: 0.0726 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0941 | LR: 1.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 8.8081s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank4 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank4 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0885 | Regression_eta: 0.0328 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0506 | LR: 1.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7999s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank4 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank4 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0843 | Regression_eta: 0.0715 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0771 | LR: 1.45e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5854s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank4 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank4 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.2325 | Regression_cos_phi: 0.0506 | Regression_energy: 0.1000 | Regression_eta: 0.0502 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0463 | LR: 1.44e-04 | DataLoad Time: 0.0134s | Model Forward Time: 4.4277s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank4 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank4 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0915 | Regression_eta: 0.0448 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0866 | LR: 1.43e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9445s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank4 [2026-10-29 20:18:48,627] INFO : Running validation on rank4 +rank4 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 20:19:38,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:19:39,385] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 20:20:09,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 20:20:10,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 20:20:11,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 20:20:12,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 20:20:22,736] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:20:23,703] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 20:20:40,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 20:20:41,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 20:20:42,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 20:20:43,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 20:20:53,513] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:20:53,999] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 20:21:09,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 20:21:10,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 20:21:11,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 20:21:11,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 20:21:21,765] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank4 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0741 | Regression_eta: 0.0627 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0551 | LR: 1.42e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.2844s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank4 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank4 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2337 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1009 | Regression_eta: 0.0796 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0884 | LR: 1.40e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1809s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank4 [2026-10-29 20:33:23,252] INFO : Step 43300/50000 rank4 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0898 | Regression_eta: 0.0432 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0610 | LR: 1.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3870s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank4 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank4 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0762 | Regression_eta: 0.0533 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0670 | LR: 1.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5486s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank4 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank4 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2004 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0883 | Regression_eta: 0.0383 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0812 | LR: 1.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1831s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank4 [2026-10-29 20:44:45,321] INFO : Step 43600/50000 rank4 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2087 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0745 | Regression_eta: 0.0443 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0487 | LR: 1.36e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0822s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank4 [2026-10-29 20:48:25,993] INFO : Step 43700/50000 rank4 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0972 | Regression_eta: 0.0430 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0696 | LR: 1.35e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.4624s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank4 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank4 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0821 | Regression_eta: 0.0500 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0769 | LR: 1.34e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.5364s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank4 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank4 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2109 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0923 | Regression_eta: 0.0708 | Regression_pt: 0.0978 | Regression_sin_phi: 0.1016 | LR: 1.33e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.3491s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank4 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank4 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0275 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0657 | Regression_eta: 0.0852 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0675 | LR: 1.32e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank4 [2026-10-29 21:00:02,713] INFO : Running validation on rank4 +rank4 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:00:57,489] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:00:58,022] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 21:01:24,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:01:25,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:01:26,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:01:27,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:01:37,630] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:01:38,954] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 21:01:54,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:01:55,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:01:56,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:01:57,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:02:07,664] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:02:08,957] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 21:02:24,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:02:25,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:02:25,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:02:26,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:02:36,616] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 21:06:42,682] INFO : Step 44100/50000 rank4 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0841 | Regression_eta: 0.0584 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0729 | LR: 1.31e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1690s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank4 [2026-10-29 21:10:40,737] INFO : Step 44200/50000 rank4 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0892 | Regression_eta: 0.0592 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0815 | LR: 1.30e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9776s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank4 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank4 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0784 | Regression_eta: 0.0450 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0652 | LR: 1.29e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1188s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank4 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank4 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0890 | Regression_eta: 0.0404 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0591 | LR: 1.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7692s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank4 [2026-10-29 21:22:16,363] INFO : Step 44500/50000 rank4 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0827 | Regression_eta: 0.0671 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0602 | LR: 1.27e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.8098s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank4 [2026-10-29 21:26:10,595] INFO : Step 44600/50000 rank4 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0819 | Regression_eta: 0.0534 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0765 | LR: 1.26e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1495s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank4 [2026-10-29 21:30:16,075] INFO : Step 44700/50000 rank4 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0880 | Regression_eta: 0.0416 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0696 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6988s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank4 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank4 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0702 | Regression_eta: 0.0422 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0556 | LR: 1.24e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.4085s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank4 [2026-10-29 21:38:11,006] INFO : Step 44900/50000 rank4 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0781 | Regression_eta: 0.0477 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0586 | LR: 1.23e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9533s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank4 [2026-10-29 21:42:13,638] INFO : Step 45000/50000 rank4 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0801 | Regression_eta: 0.0544 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0757 | LR: 1.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0113s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank4 [2026-10-29 21:42:14,558] INFO : Running validation on rank4 +rank4 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:43:06,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:43:07,017] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 21:43:32,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:43:33,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:43:34,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:43:36,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:43:46,317] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 21:43:47,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:43:47,927] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 21:44:04,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:44:04,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:44:05,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:44:06,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:44:16,889] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:44:17,711] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 21:44:33,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:44:34,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:44:34,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:44:35,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:44:45,524] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 21:48:55,616] INFO : Step 45100/50000 rank4 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0858 | Regression_eta: 0.0847 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0715 | LR: 1.21e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4102s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank4 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank4 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0747 | Regression_eta: 0.0708 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0927 | LR: 1.20e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1531s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank4 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank4 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.2440 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0980 | Regression_eta: 0.0626 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0665 | LR: 1.19e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.0132s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank4 [2026-10-29 22:00:32,929] INFO : Step 45400/50000 rank4 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0863 | Regression_eta: 0.0431 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0488 | LR: 1.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9049s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank4 [2026-10-29 22:04:23,232] INFO : Step 45500/50000 rank4 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0896 | Regression_eta: 0.0652 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0699 | LR: 1.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6360s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank4 [2026-10-29 22:08:21,194] INFO : Step 45600/50000 rank4 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2438 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0932 | Regression_eta: 0.0700 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0692 | LR: 1.17e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0167s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank4 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank4 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0779 | Regression_eta: 0.0550 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0892 | LR: 1.16e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0841s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank4 [2026-10-29 22:16:08,603] INFO : Step 45800/50000 rank4 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0861 | Regression_eta: 0.0427 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0570 | LR: 1.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6684s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank4 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank4 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0701 | Regression_eta: 0.0364 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0368 | LR: 1.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9610s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank4 [2026-10-29 22:24:24,449] INFO : Step 46000/50000 rank4 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0852 | Regression_eta: 0.0408 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0351 | LR: 1.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3014s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank4 [2026-10-29 22:24:25,403] INFO : Running validation on rank4 +rank4 [2026-10-29 22:24:25,403] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 22:25:16,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:25:17,231] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 22:25:46,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 22:25:47,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 22:25:48,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 22:25:49,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 22:25:59,516] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 22:25:59,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:26:00,492] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 22:26:16,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 22:26:17,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 22:26:19,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 22:26:20,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 22:26:30,993] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:26:31,920] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 22:26:47,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 22:26:48,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 22:26:49,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 22:26:50,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 22:27:00,020] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 22:31:16,603] INFO : Step 46100/50000 rank4 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0801 | Regression_eta: 0.0476 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0514 | LR: 1.13e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0430s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank4 [2026-10-29 22:35:12,809] INFO : Step 46200/50000 rank4 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0784 | Regression_eta: 0.0487 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0623 | LR: 1.13e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9299s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank4 [2026-10-29 22:39:25,150] INFO : Step 46300/50000 rank4 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2381 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0801 | Regression_eta: 0.0692 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0805 | LR: 1.12e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9844s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank4 [2026-10-29 22:43:23,784] INFO : Step 46400/50000 rank4 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0916 | Regression_eta: 0.0600 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0545 | LR: 1.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6671s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank4 [2026-10-29 22:47:17,351] INFO : Step 46500/50000 rank4 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0670 | Regression_eta: 0.0497 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0563 | LR: 1.11e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.5733s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank4 [2026-10-29 22:51:20,583] INFO : Step 46600/50000 rank4 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0684 | Regression_eta: 0.0518 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0723 | LR: 1.10e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.3978s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank4 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank4 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1916 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0904 | Regression_eta: 0.0501 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0559 | LR: 1.10e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0463s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank4 [2026-10-29 22:59:26,804] INFO : Step 46800/50000 rank4 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0901 | Regression_eta: 0.0464 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0619 | LR: 1.09e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.9203s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank4 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank4 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1921 | Regression_cos_phi: 0.1049 | Regression_energy: 0.0970 | Regression_eta: 0.0620 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0914 | LR: 1.09e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.0378s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank4 [2026-10-29 23:08:15,439] INFO : Step 47000/50000 rank4 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0925 | Regression_eta: 0.0569 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0714 | LR: 1.08e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2712s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank4 [2026-10-29 23:08:16,507] INFO : Running validation on rank4 +rank4 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 23:09:07,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:09:08,184] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 23:09:36,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 23:09:38,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 23:09:41,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 23:09:42,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 23:09:52,833] INFO : Time taken to make predictions on device 4 is: 0.32 min +rank4 [2026-10-29 23:09:55,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:09:55,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 23:10:11,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 23:10:12,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 23:10:13,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 23:10:14,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 23:10:24,499] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:10:26,035] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 23:10:41,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 23:10:42,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 23:10:43,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 23:10:43,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 23:10:53,793] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank4 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0774 | Regression_eta: 0.0598 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0652 | LR: 1.07e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7050s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank4 [2026-10-29 23:19:19,769] INFO : Step 47200/50000 rank4 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0944 | Regression_eta: 0.0497 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0678 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.6710s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank4 [2026-10-29 23:23:31,491] INFO : Step 47300/50000 rank4 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2133 | Regression_cos_phi: 0.1253 | Regression_energy: 0.0832 | Regression_eta: 0.0657 | Regression_pt: 0.0881 | Regression_sin_phi: 0.1060 | LR: 1.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7136s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank4 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank4 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.2004 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0734 | Regression_eta: 0.0516 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0660 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0203s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank4 [2026-10-29 23:31:38,583] INFO : Step 47500/50000 rank4 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1262 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0901 | Regression_eta: 0.0356 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0456 | LR: 1.06e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8364s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank4 [2026-10-29 23:35:34,354] INFO : Step 47600/50000 rank4 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.2024 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0873 | Regression_eta: 0.0499 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0808 | LR: 1.05e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5013s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank4 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank4 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0851 | Regression_eta: 0.0404 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0415 | LR: 1.05e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8409s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank4 [2026-10-29 23:43:53,969] INFO : Step 47800/50000 rank4 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0606 | Regression_eta: 0.0474 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0901 | LR: 1.04e-04 | DataLoad Time: 0.0133s | Model Forward Time: 4.6106s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank4 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank4 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0950 | Regression_energy: 0.0948 | Regression_eta: 0.0613 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0995 | LR: 1.04e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1960s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank4 [2026-10-29 23:51:38,727] INFO : Step 48000/50000 rank4 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0768 | Regression_eta: 0.0653 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0618 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2963s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank4 [2026-10-29 23:51:39,732] INFO : Running validation on rank4 +rank4 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 23:52:31,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:52:31,694] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 23:52:58,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 23:53:00,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 23:53:01,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 23:53:02,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 23:53:12,446] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 23:53:13,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 23:53:29,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 23:53:30,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 23:53:31,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 23:53:32,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 23:53:42,631] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:53:43,872] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 23:53:59,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 23:54:00,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 23:54:00,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 23:54:01,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 23:54:11,397] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 00:00:53,531] INFO : Step 48100/50000 rank4 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0709 | Regression_eta: 0.0445 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0702 | LR: 1.03e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5279s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank4 [2026-10-30 00:04:40,584] INFO : Step 48200/50000 rank4 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0786 | Regression_eta: 0.0592 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0941 | LR: 1.03e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0289s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank4 [2026-10-30 00:08:31,629] INFO : Step 48300/50000 rank4 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0856 | Regression_eta: 0.0439 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0545 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9454s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank4 [2026-10-30 00:12:19,574] INFO : Step 48400/50000 rank4 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0759 | Regression_eta: 0.0419 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0659 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3336s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank4 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank4 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0848 | Regression_eta: 0.0615 | Regression_pt: 0.0885 | Regression_sin_phi: 0.1173 | LR: 1.02e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.5895s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank4 [2026-10-30 00:19:50,481] INFO : Step 48600/50000 rank4 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0706 | Regression_eta: 0.0465 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0695 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9682s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank4 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank4 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0729 | Regression_eta: 0.0476 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0635 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0918s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank4 [2026-10-30 00:27:30,341] INFO : Step 48800/50000 rank4 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0673 | Regression_eta: 0.0453 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0715 | LR: 1.01e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4230s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank4 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank4 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0726 | Regression_eta: 0.0549 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0732 | LR: 1.01e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.6410s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank4 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank4 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.2299 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0700 | Regression_eta: 0.0502 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0873 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.5994s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank4 [2026-10-30 00:35:15,047] INFO : Running validation on rank4 +rank4 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:36:03,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 00:36:29,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:36:31,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:36:32,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:36:33,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:36:43,601] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-30 00:36:48,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:36:48,746] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 00:37:04,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:37:05,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:37:06,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:37:07,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:37:17,280] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:37:18,485] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 00:37:33,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 00:37:34,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 00:37:34,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 00:37:35,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 00:37:45,626] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-30 00:41:49,929] INFO : Step 49100/50000 rank4 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0881 | Regression_eta: 0.0483 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0586 | LR: 1.01e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6175s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank4 [2026-10-30 00:45:59,115] INFO : Step 49200/50000 rank4 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0721 | Regression_eta: 0.0376 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0491 | LR: 1.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5917s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank4 [2026-10-30 00:49:54,420] INFO : Step 49300/50000 rank4 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0839 | Regression_eta: 0.0550 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0929 | LR: 1.00e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9863s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank4 [2026-10-30 00:53:44,254] INFO : Step 49400/50000 rank4 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2030 | Regression_cos_phi: 0.1267 | Regression_energy: 0.0943 | Regression_eta: 0.0641 | Regression_pt: 0.0980 | Regression_sin_phi: 0.1109 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9616s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank4 [2026-10-30 00:57:39,408] INFO : Step 49500/50000 rank4 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0785 | Regression_eta: 0.0405 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0619 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0110s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank4 [2026-10-30 01:01:39,140] INFO : Step 49600/50000 rank4 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0693 | Regression_eta: 0.0453 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0674 | LR: 1.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8022s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank4 [2026-10-30 01:05:30,840] INFO : Step 49700/50000 rank4 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0762 | Regression_eta: 0.0546 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0731 | LR: 1.00e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4569s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank4 [2026-10-30 01:09:21,932] INFO : Step 49800/50000 rank4 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0671 | Regression_eta: 0.0422 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0643 | LR: 1.00e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3593s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank4 [2026-10-30 01:13:13,492] INFO : Step 49900/50000 rank4 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0878 | Regression_eta: 0.0660 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0673 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0840s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank4 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank4 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2206 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0781 | Regression_eta: 0.0535 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0665 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8266s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank4 [2026-10-30 01:17:09,619] INFO : Running validation on rank4 +rank4 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 01:18:02,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:18:03,251] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 01:18:29,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 01:18:30,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 01:18:31,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 01:18:32,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 01:18:42,172] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:18:42,590] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:18:43,093] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 01:18:59,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 01:19:00,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 01:19:01,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 01:19:02,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 01:19:12,141] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 01:19:28,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:19:29,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:19:29,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:19:30,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:19:40,600] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-30 01:19:41,605] INFO : Training completed. Total time on device 4: 629.367min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.5 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..44b2b55bee3271d3b5d11f3f0ca7b13e7719184d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.5 @@ -0,0 +1,2483 @@ +rank5 [2026-10-28 11:12:33,082] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-28 11:12:33,083] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank5 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-28 11:12:57,453] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-28 11:12:57,453] INFO : nn0 parameters: 171776 +rank5 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank5 [2026-10-28 11:12:57,454] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 1 +rank5 [2026-10-28 11:12:57,459] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,464] INFO : Initializing backbone layer 2 +rank5 [2026-10-28 11:12:57,465] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,470] INFO : Initializing backbone layer 3 +rank5 [2026-10-28 11:12:57,470] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,475] INFO : Initializing backbone layer 4 +rank5 [2026-10-28 11:12:57,475] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,480] INFO : Initializing backbone layer 5 +rank5 [2026-10-28 11:12:57,480] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:57,485] INFO : Backbone initialization took 0.03s +rank5 [2026-10-28 11:12:57,486] INFO : backbone parameters: 2374656 +rank5 [2026-10-28 11:12:57,486] INFO : Initializing output DNNs +rank5 [2026-10-28 11:12:57,524] INFO : Output DNNs initialization took 0.04s +rank5 [2026-10-28 11:12:57,524] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-28 11:12:57,524] INFO : jagged_attention enabled=True +rank5 [2026-10-28 11:12:57,524] INFO : flash_attn_varlen enabled=True +rank5 [2026-10-28 11:12:57,524] INFO : task_queries enabled=False +rank5 [2026-10-28 11:12:57,524] INFO : classification_norm parameters: 0 +rank5 [2026-10-28 11:12:57,524] INFO : regression_norm parameters: 0 +rank5 [2026-10-28 11:12:57,524] INFO : classification_query parameters: 0 +rank5 [2026-10-28 11:12:57,524] INFO : regression_query parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : classification_readout parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : regression_readout parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : output_mode=set +rank5 [2026-10-28 11:12:57,525] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-28 11:12:57,525] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_pid parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_pt parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_eta parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-28 11:12:57,525] INFO : nn_energy parameters: 0 +rank5 [2026-10-28 11:12:57,526] INFO : Total MLPF parameters: 7260244 +rank5 [2026-10-28 11:12:57,526] INFO : MLPF __init__ done +rank5 [2026-10-28 11:12:57,526] INFO : Instantiated model +rank5 [2026-10-28 11:12:57,526] INFO : Moving model to device rank=5 +rank5 [2026-10-28 11:12:57,561] INFO : Moved model to device rank=5 +rank5 [2026-10-28 11:12:57,562] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-28 11:12:57,562] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-28 11:12:57,814] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank5 [2026-10-28 11:12:57,814] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-28 11:12:57,814] INFO : Trainable parameters: 7260244 +rank5 [2026-10-28 11:12:57,814] INFO : Non-trainable parameters: 0 +rank5 [2026-10-28 11:12:57,814] INFO : Total parameters: 7260244 +rank5 [2026-10-28 11:12:57,817] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-28 11:12:57,817] INFO : Getting dataloaders +rank5 [2026-10-28 11:12:57,817] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:03,704] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:05,061] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,272] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,549] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,813] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-28 11:13:06,814] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-28 11:13:20,782] INFO : Got dataloaders +rank5 [2026-10-28 11:13:20,783] INFO : loader split=train rank=5 len=21108 +rank5 [2026-10-28 11:13:20,783] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-28 11:13:20,783] INFO : Starting training from step 1 to 50000 on rank 5 of 8 +rank5 [2026-10-28 11:13:20,783] INFO : Creating train_iterator +rank5 [2026-10-28 11:13:20,783] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-28 11:13:35,369] INFO : Created train_iterator +rank5 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank5 [2026-10-28 11:17:20,490] INFO : Step 100/50000 rank5 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.9397 | Classification_binary: 0.4850 | Regression_cos_phi: 0.0105 | Regression_energy: 1.1829 | Regression_eta: 0.0282 | Regression_pt: 1.0841 | Regression_sin_phi: 0.0096 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.1323s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank5 [2026-10-28 11:20:57,689] INFO : Step 200/50000 rank5 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4653 | Classification_binary: 0.5515 | Regression_cos_phi: 0.3853 | Regression_energy: 0.3892 | Regression_eta: 0.3130 | Regression_pt: 0.4046 | Regression_sin_phi: 0.3941 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 3.5595s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank5 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank5 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3493 | Classification_binary: 0.4900 | Regression_cos_phi: 0.3473 | Regression_energy: 0.3004 | Regression_eta: 0.3066 | Regression_pt: 0.3049 | Regression_sin_phi: 0.4006 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 3.7565s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank5 [2026-10-28 11:28:09,551] INFO : Step 400/50000 rank5 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3016 | Classification_binary: 0.4283 | Regression_cos_phi: 0.3738 | Regression_energy: 0.3375 | Regression_eta: 0.2756 | Regression_pt: 0.3481 | Regression_sin_phi: 0.3468 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 3.5806s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank5 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank5 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2802 | Classification_binary: 0.5131 | Regression_cos_phi: 0.2860 | Regression_energy: 0.2981 | Regression_eta: 0.3023 | Regression_pt: 0.3197 | Regression_sin_phi: 0.3154 | LR: 1.00e-03 | DataLoad Time: 0.0153s | Model Forward Time: 4.0268s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank5 [2026-10-28 11:35:32,722] INFO : Step 600/50000 rank5 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2570 | Classification_binary: 0.4965 | Regression_cos_phi: 0.3530 | Regression_energy: 0.2818 | Regression_eta: 0.2836 | Regression_pt: 0.3020 | Regression_sin_phi: 0.3673 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.1147s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank5 [2026-10-28 11:39:09,492] INFO : Step 700/50000 rank5 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2492 | Classification_binary: 0.4007 | Regression_cos_phi: 0.2781 | Regression_energy: 0.2573 | Regression_eta: 0.2511 | Regression_pt: 0.2621 | Regression_sin_phi: 0.2448 | LR: 1.00e-03 | DataLoad Time: 0.0176s | Model Forward Time: 3.3981s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank5 [2026-10-28 11:42:49,057] INFO : Step 800/50000 rank5 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2213 | Classification_binary: 0.4156 | Regression_cos_phi: 0.1938 | Regression_energy: 0.3090 | Regression_eta: 0.1721 | Regression_pt: 0.3226 | Regression_sin_phi: 0.2036 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.6492s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank5 [2026-10-28 11:46:31,921] INFO : Step 900/50000 rank5 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2307 | Classification_binary: 0.5156 | Regression_cos_phi: 0.3206 | Regression_energy: 0.3174 | Regression_eta: 0.2476 | Regression_pt: 0.3296 | Regression_sin_phi: 0.2126 | LR: 9.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5690s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank5 [2026-10-28 11:50:09,949] INFO : Step 1000/50000 rank5 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1900 | Classification_binary: 0.4411 | Regression_cos_phi: 0.2533 | Regression_energy: 0.2898 | Regression_eta: 0.2364 | Regression_pt: 0.2981 | Regression_sin_phi: 0.2202 | LR: 9.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.4170s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank5 [2026-10-28 11:50:10,911] INFO : Running validation on rank5 +rank5 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 11:50:58,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:50:59,086] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 11:51:19,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 11:51:20,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 11:51:21,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 11:51:22,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 11:51:32,181] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 11:51:32,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:51:33,312] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 11:51:49,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 11:51:50,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 11:51:51,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 11:51:52,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 11:52:02,074] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 11:52:02,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:52:03,469] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 11:52:18,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 11:52:19,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 11:52:20,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 11:52:20,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 11:52:30,768] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 11:56:43,806] INFO : Step 1100/50000 rank5 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2141 | Classification_binary: 0.4339 | Regression_cos_phi: 0.2262 | Regression_energy: 0.2765 | Regression_eta: 0.2546 | Regression_pt: 0.2853 | Regression_sin_phi: 0.2312 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9226s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank5 [2026-10-28 12:00:30,890] INFO : Step 1200/50000 rank5 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2438 | Classification_binary: 0.4851 | Regression_cos_phi: 0.2291 | Regression_energy: 0.2757 | Regression_eta: 0.2355 | Regression_pt: 0.2833 | Regression_sin_phi: 0.2455 | LR: 9.99e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.3993s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank5 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank5 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2053 | Classification_binary: 0.4200 | Regression_cos_phi: 0.2283 | Regression_energy: 0.2403 | Regression_eta: 0.2072 | Regression_pt: 0.2495 | Regression_sin_phi: 0.2234 | LR: 9.98e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1261s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank5 [2026-10-28 12:08:10,852] INFO : Step 1400/50000 rank5 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2048 | Classification_binary: 0.4247 | Regression_cos_phi: 0.2368 | Regression_energy: 0.2779 | Regression_eta: 0.1869 | Regression_pt: 0.2859 | Regression_sin_phi: 0.2028 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7549s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank5 [2026-10-28 12:11:57,637] INFO : Step 1500/50000 rank5 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1755 | Classification_binary: 0.4428 | Regression_cos_phi: 0.2364 | Regression_energy: 0.3103 | Regression_eta: 0.1925 | Regression_pt: 0.3195 | Regression_sin_phi: 0.2287 | LR: 9.98e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5807s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank5 [2026-10-28 12:15:43,592] INFO : Step 1600/50000 rank5 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2482 | Classification_binary: 0.4660 | Regression_cos_phi: 0.2761 | Regression_energy: 0.2652 | Regression_eta: 0.2166 | Regression_pt: 0.2781 | Regression_sin_phi: 0.3287 | LR: 9.98e-04 | DataLoad Time: 0.0248s | Model Forward Time: 4.0353s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank5 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank5 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.3747 | Regression_cos_phi: 0.2640 | Regression_energy: 0.2190 | Regression_eta: 0.1909 | Regression_pt: 0.2265 | Regression_sin_phi: 0.2228 | LR: 9.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.3912s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank5 [2026-10-28 12:23:17,460] INFO : Step 1800/50000 rank5 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2323 | Classification_binary: 0.4048 | Regression_cos_phi: 0.1963 | Regression_energy: 0.2196 | Regression_eta: 0.2186 | Regression_pt: 0.2322 | Regression_sin_phi: 0.2083 | LR: 9.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9409s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank5 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank5 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2146 | Classification_binary: 0.4642 | Regression_cos_phi: 0.2190 | Regression_energy: 0.2363 | Regression_eta: 0.1993 | Regression_pt: 0.2489 | Regression_sin_phi: 0.1898 | LR: 9.97e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5709s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank5 [2026-10-28 12:30:59,047] INFO : Step 2000/50000 rank5 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1808 | Classification_binary: 0.3935 | Regression_cos_phi: 0.2366 | Regression_energy: 0.2403 | Regression_eta: 0.1909 | Regression_pt: 0.2505 | Regression_sin_phi: 0.1593 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1434s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank5 [2026-10-28 12:31:00,064] INFO : Running validation on rank5 +rank5 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 12:31:47,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:31:48,296] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 12:32:13,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:32:14,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:32:15,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:32:16,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:32:26,430] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 12:32:27,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:32:27,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 12:32:43,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:32:44,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:32:45,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:32:46,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:32:56,722] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 12:32:57,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:32:57,640] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 12:33:13,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:33:14,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:33:14,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:33:15,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:33:25,578] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 12:37:32,605] INFO : Step 2100/50000 rank5 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2189 | Classification_binary: 0.3659 | Regression_cos_phi: 0.2272 | Regression_energy: 0.3016 | Regression_eta: 0.2122 | Regression_pt: 0.3157 | Regression_sin_phi: 0.1891 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7723s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank5 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank5 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2183 | Classification_binary: 0.4500 | Regression_cos_phi: 0.3358 | Regression_energy: 0.2739 | Regression_eta: 0.2404 | Regression_pt: 0.2924 | Regression_sin_phi: 0.2747 | LR: 9.96e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7067s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank5 [2026-10-28 12:45:14,507] INFO : Step 2300/50000 rank5 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2151 | Classification_binary: 0.4082 | Regression_cos_phi: 0.3022 | Regression_energy: 0.2417 | Regression_eta: 0.2502 | Regression_pt: 0.2544 | Regression_sin_phi: 0.2905 | LR: 9.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9878s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank5 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank5 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1768 | Classification_binary: 0.3475 | Regression_cos_phi: 0.1898 | Regression_energy: 0.2583 | Regression_eta: 0.1846 | Regression_pt: 0.2647 | Regression_sin_phi: 0.2212 | LR: 9.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.4496s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank5 [2026-10-28 12:52:58,132] INFO : Step 2500/50000 rank5 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2552 | Classification_binary: 0.4526 | Regression_cos_phi: 0.3046 | Regression_energy: 0.2473 | Regression_eta: 0.2683 | Regression_pt: 0.2618 | Regression_sin_phi: 0.2958 | LR: 9.94e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3992s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank5 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank5 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2107 | Classification_binary: 0.4585 | Regression_cos_phi: 0.2803 | Regression_energy: 0.2471 | Regression_eta: 0.2743 | Regression_pt: 0.2578 | Regression_sin_phi: 0.2566 | LR: 9.94e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2555s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank5 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank5 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1810 | Classification_binary: 0.3603 | Regression_cos_phi: 0.2190 | Regression_energy: 0.2386 | Regression_eta: 0.2135 | Regression_pt: 0.2451 | Regression_sin_phi: 0.2845 | LR: 9.94e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0881s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank5 [2026-10-28 13:04:27,894] INFO : Step 2800/50000 rank5 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1704 | Classification_binary: 0.3675 | Regression_cos_phi: 0.2355 | Regression_energy: 0.2124 | Regression_eta: 0.2087 | Regression_pt: 0.2264 | Regression_sin_phi: 0.2277 | LR: 9.93e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1950s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank5 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank5 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1945 | Classification_binary: 0.4818 | Regression_cos_phi: 0.2411 | Regression_energy: 0.2220 | Regression_eta: 0.2653 | Regression_pt: 0.2348 | Regression_sin_phi: 0.2602 | LR: 9.93e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1946s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank5 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank5 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2053 | Classification_binary: 0.3895 | Regression_cos_phi: 0.2592 | Regression_energy: 0.2239 | Regression_eta: 0.2226 | Regression_pt: 0.2358 | Regression_sin_phi: 0.2457 | LR: 9.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5276s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank5 [2026-10-28 13:12:10,496] INFO : Running validation on rank5 +rank5 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:13:02,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:13:03,432] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 13:13:30,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:13:31,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:13:31,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:13:32,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:13:42,829] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:13:44,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 13:14:00,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:14:01,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:14:02,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:14:03,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:14:13,166] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:14:14,291] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 13:14:29,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:14:30,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:14:31,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:14:31,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:14:41,761] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 13:19:02,742] INFO : Step 3100/50000 rank5 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1804 | Classification_binary: 0.3383 | Regression_cos_phi: 0.1872 | Regression_energy: 0.1839 | Regression_eta: 0.1454 | Regression_pt: 0.1982 | Regression_sin_phi: 0.1674 | LR: 9.91e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.5825s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank5 [2026-10-28 13:23:00,672] INFO : Step 3200/50000 rank5 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1958 | Classification_binary: 0.3696 | Regression_cos_phi: 0.2043 | Regression_energy: 0.2452 | Regression_eta: 0.1952 | Regression_pt: 0.2554 | Regression_sin_phi: 0.2237 | LR: 9.91e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8508s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank5 [2026-10-28 13:26:54,112] INFO : Step 3300/50000 rank5 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1807 | Classification_binary: 0.3962 | Regression_cos_phi: 0.2320 | Regression_energy: 0.2286 | Regression_eta: 0.2366 | Regression_pt: 0.2397 | Regression_sin_phi: 0.2703 | LR: 9.90e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7306s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank5 [2026-10-28 13:30:48,597] INFO : Step 3400/50000 rank5 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1582 | Classification_binary: 0.2885 | Regression_cos_phi: 0.1569 | Regression_energy: 0.2196 | Regression_eta: 0.1604 | Regression_pt: 0.2274 | Regression_sin_phi: 0.1914 | LR: 9.90e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0754s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank5 [2026-10-28 13:34:47,771] INFO : Step 3500/50000 rank5 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2014 | Classification_binary: 0.3879 | Regression_cos_phi: 0.2277 | Regression_energy: 0.1973 | Regression_eta: 0.2108 | Regression_pt: 0.2058 | Regression_sin_phi: 0.2521 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6681s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank5 [2026-10-28 13:38:46,123] INFO : Step 3600/50000 rank5 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1460 | Classification_binary: 0.4031 | Regression_cos_phi: 0.1968 | Regression_energy: 0.2012 | Regression_eta: 0.1874 | Regression_pt: 0.2082 | Regression_sin_phi: 0.1943 | LR: 9.89e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.1032s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank5 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank5 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1656 | Classification_binary: 0.3420 | Regression_cos_phi: 0.2259 | Regression_energy: 0.2196 | Regression_eta: 0.2013 | Regression_pt: 0.2195 | Regression_sin_phi: 0.2143 | LR: 9.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2045s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank5 [2026-10-28 13:46:34,310] INFO : Step 3800/50000 rank5 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1858 | Classification_binary: 0.3231 | Regression_cos_phi: 0.1905 | Regression_energy: 0.1918 | Regression_eta: 0.1963 | Regression_pt: 0.2019 | Regression_sin_phi: 0.2369 | LR: 9.87e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.9858s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank5 [2026-10-28 13:50:23,156] INFO : Step 3900/50000 rank5 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1815 | Classification_binary: 0.4492 | Regression_cos_phi: 0.2568 | Regression_energy: 0.2077 | Regression_eta: 0.2024 | Regression_pt: 0.2173 | Regression_sin_phi: 0.2237 | LR: 9.87e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2423s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank5 [2026-10-28 13:54:11,925] INFO : Step 4000/50000 rank5 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1619 | Classification_binary: 0.4002 | Regression_cos_phi: 0.2429 | Regression_energy: 0.1974 | Regression_eta: 0.2002 | Regression_pt: 0.2092 | Regression_sin_phi: 0.1890 | LR: 9.86e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5800s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank5 [2026-10-28 13:54:12,844] INFO : Running validation on rank5 +rank5 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:55:03,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 13:55:33,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:55:34,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:55:34,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:55:35,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:55:45,794] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:55:46,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 13:56:03,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:56:04,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:56:05,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:56:06,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:56:16,112] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:56:16,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:56:16,986] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 13:56:32,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:56:33,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:56:33,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:56:34,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:56:44,569] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank5 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 0.2855 | Regression_cos_phi: 0.1933 | Regression_energy: 0.2070 | Regression_eta: 0.1480 | Regression_pt: 0.2168 | Regression_sin_phi: 0.1882 | LR: 9.85e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9532s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank5 [2026-10-28 14:04:50,728] INFO : Step 4200/50000 rank5 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1918 | Classification_binary: 0.3507 | Regression_cos_phi: 0.2055 | Regression_energy: 0.2388 | Regression_eta: 0.1839 | Regression_pt: 0.2513 | Regression_sin_phi: 0.2032 | LR: 9.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4864s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank5 [2026-10-28 14:08:42,269] INFO : Step 4300/50000 rank5 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1762 | Classification_binary: 0.3785 | Regression_cos_phi: 0.2169 | Regression_energy: 0.2454 | Regression_eta: 0.2033 | Regression_pt: 0.2568 | Regression_sin_phi: 0.2075 | LR: 9.84e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1053s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank5 [2026-10-28 14:12:29,193] INFO : Step 4400/50000 rank5 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1209 | Classification_binary: 0.2900 | Regression_cos_phi: 0.2004 | Regression_energy: 0.2087 | Regression_eta: 0.1784 | Regression_pt: 0.2143 | Regression_sin_phi: 0.1688 | LR: 9.83e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.7760s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank5 [2026-10-28 14:16:26,826] INFO : Step 4500/50000 rank5 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1525 | Classification_binary: 0.3320 | Regression_cos_phi: 0.2028 | Regression_energy: 0.1989 | Regression_eta: 0.1458 | Regression_pt: 0.2069 | Regression_sin_phi: 0.2528 | LR: 9.82e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4663s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank5 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank5 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1492 | Classification_binary: 0.4077 | Regression_cos_phi: 0.2449 | Regression_energy: 0.2032 | Regression_eta: 0.1956 | Regression_pt: 0.2117 | Regression_sin_phi: 0.2494 | LR: 9.81e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1358s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank5 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank5 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1602 | Classification_binary: 0.3788 | Regression_cos_phi: 0.1760 | Regression_energy: 0.1896 | Regression_eta: 0.1972 | Regression_pt: 0.1996 | Regression_sin_phi: 0.2705 | LR: 9.81e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3030s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank5 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank5 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1370 | Classification_binary: 0.3248 | Regression_cos_phi: 0.1768 | Regression_energy: 0.1802 | Regression_eta: 0.1576 | Regression_pt: 0.1889 | Regression_sin_phi: 0.1956 | LR: 9.80e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9654s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank5 [2026-10-28 14:32:01,138] INFO : Step 4900/50000 rank5 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.4015 | Regression_cos_phi: 0.3051 | Regression_energy: 0.1936 | Regression_eta: 0.2754 | Regression_pt: 0.2017 | Regression_sin_phi: 0.2663 | LR: 9.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8452s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank5 [2026-10-28 14:35:52,795] INFO : Step 5000/50000 rank5 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1452 | Classification_binary: 0.3676 | Regression_cos_phi: 0.1906 | Regression_energy: 0.2011 | Regression_eta: 0.1894 | Regression_pt: 0.2105 | Regression_sin_phi: 0.1506 | LR: 9.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8405s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank5 [2026-10-28 14:35:53,776] INFO : Running validation on rank5 +rank5 [2026-10-28 14:35:53,777] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 14:35:53,777] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 14:36:42,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:36:42,706] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 14:37:08,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 14:37:09,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 14:37:10,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 14:37:11,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 14:37:21,760] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:37:22,910] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 14:37:38,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 14:37:39,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 14:37:40,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 14:37:41,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 14:37:51,686] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 14:37:52,214] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:37:52,791] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 14:38:08,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 14:38:08,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 14:38:09,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 14:38:10,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 14:38:20,228] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank5 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1230 | Classification_binary: 0.2833 | Regression_cos_phi: 0.1536 | Regression_energy: 0.2391 | Regression_eta: 0.1601 | Regression_pt: 0.2454 | Regression_sin_phi: 0.1819 | LR: 9.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3391s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank5 [2026-10-28 14:46:36,780] INFO : Step 5200/50000 rank5 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1616 | Classification_binary: 0.3685 | Regression_cos_phi: 0.2012 | Regression_energy: 0.1975 | Regression_eta: 0.2015 | Regression_pt: 0.2108 | Regression_sin_phi: 0.1987 | LR: 9.76e-04 | DataLoad Time: 0.0136s | Model Forward Time: 3.9425s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank5 [2026-10-28 14:50:33,436] INFO : Step 5300/50000 rank5 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.3871 | Regression_cos_phi: 0.2188 | Regression_energy: 0.1767 | Regression_eta: 0.1837 | Regression_pt: 0.1856 | Regression_sin_phi: 0.2324 | LR: 9.75e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.1866s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank5 [2026-10-28 14:54:36,491] INFO : Step 5400/50000 rank5 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1187 | Classification_binary: 0.3346 | Regression_cos_phi: 0.2195 | Regression_energy: 0.1659 | Regression_eta: 0.1549 | Regression_pt: 0.1739 | Regression_sin_phi: 0.1685 | LR: 9.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.4916s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank5 [2026-10-28 14:58:42,458] INFO : Step 5500/50000 rank5 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1243 | Classification_binary: 0.3391 | Regression_cos_phi: 0.1614 | Regression_energy: 0.1880 | Regression_eta: 0.1528 | Regression_pt: 0.1988 | Regression_sin_phi: 0.1902 | LR: 9.73e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5914s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank5 [2026-10-28 15:02:44,093] INFO : Step 5600/50000 rank5 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1988 | Classification_binary: 0.4536 | Regression_cos_phi: 0.2466 | Regression_energy: 0.1934 | Regression_eta: 0.1815 | Regression_pt: 0.2044 | Regression_sin_phi: 0.2315 | LR: 9.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6394s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank5 [2026-10-28 15:06:32,821] INFO : Step 5700/50000 rank5 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.3083 | Regression_cos_phi: 0.1612 | Regression_energy: 0.1887 | Regression_eta: 0.1965 | Regression_pt: 0.1999 | Regression_sin_phi: 0.1802 | LR: 9.71e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1512s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank5 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank5 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1677 | Classification_binary: 0.3144 | Regression_cos_phi: 0.1961 | Regression_energy: 0.2235 | Regression_eta: 0.1716 | Regression_pt: 0.2300 | Regression_sin_phi: 0.2858 | LR: 9.70e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.5299s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank5 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank5 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1679 | Classification_binary: 0.4292 | Regression_cos_phi: 0.2553 | Regression_energy: 0.2030 | Regression_eta: 0.2223 | Regression_pt: 0.2104 | Regression_sin_phi: 0.2176 | LR: 9.69e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.0727s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank5 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank5 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1643 | Classification_binary: 0.3904 | Regression_cos_phi: 0.2047 | Regression_energy: 0.1959 | Regression_eta: 0.1813 | Regression_pt: 0.2057 | Regression_sin_phi: 0.2042 | LR: 9.68e-04 | DataLoad Time: 0.0121s | Model Forward Time: 6.3910s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank5 [2026-10-28 15:18:46,644] INFO : Running validation on rank5 +rank5 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:19:35,275] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 15:19:55,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 15:19:57,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 15:19:58,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 15:20:00,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 15:20:10,014] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:20:11,684] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 15:20:27,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 15:20:28,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 15:20:29,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 15:20:30,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 15:20:40,415] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:20:40,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:20:41,483] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 15:20:56,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 15:20:57,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 15:20:58,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 15:20:59,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 15:21:09,034] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 15:25:37,243] INFO : Step 6100/50000 rank5 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1127 | Classification_binary: 0.2413 | Regression_cos_phi: 0.1493 | Regression_energy: 0.1603 | Regression_eta: 0.1271 | Regression_pt: 0.1673 | Regression_sin_phi: 0.1223 | LR: 9.67e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2779s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank5 [2026-10-28 15:29:27,137] INFO : Step 6200/50000 rank5 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1282 | Classification_binary: 0.2790 | Regression_cos_phi: 0.2011 | Regression_energy: 0.1729 | Regression_eta: 0.1513 | Regression_pt: 0.1821 | Regression_sin_phi: 0.1653 | LR: 9.66e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.3967s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank5 [2026-10-28 15:33:16,837] INFO : Step 6300/50000 rank5 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1581 | Classification_binary: 0.3496 | Regression_cos_phi: 0.2087 | Regression_energy: 0.2078 | Regression_eta: 0.1714 | Regression_pt: 0.2178 | Regression_sin_phi: 0.1964 | LR: 9.65e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2455s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank5 [2026-10-28 15:37:09,470] INFO : Step 6400/50000 rank5 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.3057 | Regression_cos_phi: 0.1200 | Regression_energy: 0.2155 | Regression_eta: 0.1468 | Regression_pt: 0.2250 | Regression_sin_phi: 0.1138 | LR: 9.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9980s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank5 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank5 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1040 | Classification_binary: 0.2749 | Regression_cos_phi: 0.1549 | Regression_energy: 0.1682 | Regression_eta: 0.1241 | Regression_pt: 0.1754 | Regression_sin_phi: 0.1418 | LR: 9.63e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.6702s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank5 [2026-10-28 15:45:09,040] INFO : Step 6600/50000 rank5 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1573 | Classification_binary: 0.3705 | Regression_cos_phi: 0.1960 | Regression_energy: 0.1829 | Regression_eta: 0.1827 | Regression_pt: 0.1899 | Regression_sin_phi: 0.1765 | LR: 9.62e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2524s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank5 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank5 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.3349 | Regression_cos_phi: 0.1858 | Regression_energy: 0.1817 | Regression_eta: 0.1758 | Regression_pt: 0.1882 | Regression_sin_phi: 0.2028 | LR: 9.61e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.0311s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank5 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank5 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1644 | Classification_binary: 0.2980 | Regression_cos_phi: 0.2504 | Regression_energy: 0.1776 | Regression_eta: 0.1811 | Regression_pt: 0.1822 | Regression_sin_phi: 0.2310 | LR: 9.60e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8866s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank5 [2026-10-28 15:56:38,728] INFO : Step 6900/50000 rank5 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1632 | Classification_binary: 0.3488 | Regression_cos_phi: 0.2022 | Regression_energy: 0.1790 | Regression_eta: 0.1479 | Regression_pt: 0.1869 | Regression_sin_phi: 0.1965 | LR: 9.58e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.8814s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank5 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank5 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1391 | Classification_binary: 0.3593 | Regression_cos_phi: 0.2026 | Regression_energy: 0.1698 | Regression_eta: 0.1525 | Regression_pt: 0.1756 | Regression_sin_phi: 0.2004 | LR: 9.57e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.4716s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank5 [2026-10-28 16:00:38,673] INFO : Running validation on rank5 +rank5 [2026-10-28 16:00:38,673] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:00:38,673] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:01:32,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:01:33,504] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 16:02:00,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:02:01,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:02:02,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:02:03,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:02:13,181] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:02:14,069] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 16:02:30,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:02:31,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:02:32,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:02:33,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:02:43,490] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:02:44,278] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 16:02:59,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:03:00,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:03:01,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:03:01,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:03:11,828] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank5 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2798 | Regression_cos_phi: 0.1243 | Regression_energy: 0.1547 | Regression_eta: 0.1199 | Regression_pt: 0.1588 | Regression_sin_phi: 0.1502 | LR: 9.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5601s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank5 [2026-10-28 16:11:41,309] INFO : Step 7200/50000 rank5 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1592 | Classification_binary: 0.3810 | Regression_cos_phi: 0.2251 | Regression_energy: 0.1739 | Regression_eta: 0.1751 | Regression_pt: 0.1816 | Regression_sin_phi: 0.2327 | LR: 9.55e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9561s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank5 [2026-10-28 16:15:45,432] INFO : Step 7300/50000 rank5 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1618 | Classification_binary: 0.4002 | Regression_cos_phi: 0.2319 | Regression_energy: 0.1698 | Regression_eta: 0.1878 | Regression_pt: 0.1771 | Regression_sin_phi: 0.2004 | LR: 9.53e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.9003s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank5 [2026-10-28 16:19:47,790] INFO : Step 7400/50000 rank5 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2605 | Regression_cos_phi: 0.1460 | Regression_energy: 0.1702 | Regression_eta: 0.1028 | Regression_pt: 0.1745 | Regression_sin_phi: 0.1621 | LR: 9.52e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.6017s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank5 [2026-10-28 16:23:56,337] INFO : Step 7500/50000 rank5 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1240 | Classification_binary: 0.3139 | Regression_cos_phi: 0.1639 | Regression_energy: 0.1771 | Regression_eta: 0.1529 | Regression_pt: 0.1884 | Regression_sin_phi: 0.1915 | LR: 9.51e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7928s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank5 [2026-10-28 16:27:58,399] INFO : Step 7600/50000 rank5 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1516 | Classification_binary: 0.3620 | Regression_cos_phi: 0.1807 | Regression_energy: 0.1716 | Regression_eta: 0.1386 | Regression_pt: 0.1783 | Regression_sin_phi: 0.1922 | LR: 9.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7004s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank5 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank5 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.2859 | Regression_cos_phi: 0.1473 | Regression_energy: 0.1466 | Regression_eta: 0.1149 | Regression_pt: 0.1545 | Regression_sin_phi: 0.1644 | LR: 9.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9792s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank5 [2026-10-28 16:35:55,392] INFO : Step 7800/50000 rank5 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1456 | Classification_binary: 0.2675 | Regression_cos_phi: 0.1735 | Regression_energy: 0.1433 | Regression_eta: 0.1413 | Regression_pt: 0.1482 | Regression_sin_phi: 0.1996 | LR: 9.47e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2114s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank5 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank5 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1651 | Classification_binary: 0.3541 | Regression_cos_phi: 0.2175 | Regression_energy: 0.1543 | Regression_eta: 0.2041 | Regression_pt: 0.1610 | Regression_sin_phi: 0.2535 | LR: 9.46e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3425s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank5 [2026-10-28 16:44:12,790] INFO : Step 8000/50000 rank5 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1421 | Classification_binary: 0.3351 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1740 | Regression_eta: 0.1259 | Regression_pt: 0.1825 | Regression_sin_phi: 0.1670 | LR: 9.44e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.2428s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank5 [2026-10-28 16:44:13,937] INFO : Running validation on rank5 +rank5 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:45:04,007] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:45:04,563] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 16:45:30,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:45:31,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:45:32,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:45:33,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:45:43,887] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:45:45,512] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 16:46:01,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:46:02,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:46:03,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:46:04,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:46:14,023] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:46:15,125] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 16:46:30,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:46:31,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:46:32,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:46:32,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:46:42,765] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank5 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2336 | Regression_cos_phi: 0.1314 | Regression_energy: 0.1635 | Regression_eta: 0.1154 | Regression_pt: 0.1695 | Regression_sin_phi: 0.1503 | LR: 9.43e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7746s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank5 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank5 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1457 | Classification_binary: 0.3309 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1520 | Regression_eta: 0.1386 | Regression_pt: 0.1582 | Regression_sin_phi: 0.1771 | LR: 9.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1565s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank5 [2026-10-28 16:59:31,718] INFO : Step 8300/50000 rank5 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1318 | Classification_binary: 0.3664 | Regression_cos_phi: 0.1709 | Regression_energy: 0.1631 | Regression_eta: 0.1640 | Regression_pt: 0.1725 | Regression_sin_phi: 0.1865 | LR: 9.40e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.9199s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank5 [2026-10-28 17:03:26,315] INFO : Step 8400/50000 rank5 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2535 | Regression_cos_phi: 0.1597 | Regression_energy: 0.1507 | Regression_eta: 0.1246 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1446 | LR: 9.39e-04 | DataLoad Time: 0.0228s | Model Forward Time: 6.0445s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank5 [2026-10-28 17:07:18,317] INFO : Step 8500/50000 rank5 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1217 | Classification_binary: 0.3172 | Regression_cos_phi: 0.1849 | Regression_energy: 0.1572 | Regression_eta: 0.1328 | Regression_pt: 0.1641 | Regression_sin_phi: 0.1472 | LR: 9.37e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4974s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank5 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank5 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1274 | Classification_binary: 0.3691 | Regression_cos_phi: 0.2240 | Regression_energy: 0.1795 | Regression_eta: 0.1773 | Regression_pt: 0.1852 | Regression_sin_phi: 0.1612 | LR: 9.36e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1693s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank5 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank5 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1558 | Regression_eta: 0.0963 | Regression_pt: 0.1613 | Regression_sin_phi: 0.1514 | LR: 9.34e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank5 [2026-10-28 17:19:40,248] INFO : Step 8800/50000 rank5 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1088 | Classification_binary: 0.2553 | Regression_cos_phi: 0.1510 | Regression_energy: 0.1544 | Regression_eta: 0.1147 | Regression_pt: 0.1612 | Regression_sin_phi: 0.1633 | LR: 9.33e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.4433s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank5 [2026-10-28 17:23:38,392] INFO : Step 8900/50000 rank5 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1046 | Classification_binary: 0.3156 | Regression_cos_phi: 0.1796 | Regression_energy: 0.1877 | Regression_eta: 0.1551 | Regression_pt: 0.1992 | Regression_sin_phi: 0.1718 | LR: 9.31e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6551s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank5 [2026-10-28 17:27:33,262] INFO : Step 9000/50000 rank5 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.3496 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1544 | Regression_eta: 0.1197 | Regression_pt: 0.1636 | Regression_sin_phi: 0.1413 | LR: 9.30e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8424s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank5 [2026-10-28 17:27:34,333] INFO : Running validation on rank5 +rank5 [2026-10-28 17:27:34,334] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 17:27:34,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 17:28:24,729] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:28:25,265] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 17:28:52,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 17:28:53,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 17:28:53,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 17:28:54,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 17:29:04,928] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 17:29:05,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:29:06,070] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 17:29:22,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 17:29:23,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 17:29:25,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 17:29:26,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 17:29:36,409] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 17:29:37,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:29:37,692] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 17:29:53,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 17:29:53,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 17:29:54,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 17:29:55,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 17:30:05,378] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 17:34:23,899] INFO : Step 9100/50000 rank5 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2410 | Regression_cos_phi: 0.0945 | Regression_energy: 0.1443 | Regression_eta: 0.0877 | Regression_pt: 0.1500 | Regression_sin_phi: 0.0975 | LR: 9.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1052s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank5 [2026-10-28 17:38:24,816] INFO : Step 9200/50000 rank5 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1433 | Classification_binary: 0.3117 | Regression_cos_phi: 0.1467 | Regression_energy: 0.1396 | Regression_eta: 0.1433 | Regression_pt: 0.1469 | Regression_sin_phi: 0.1688 | LR: 9.27e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3275s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank5 [2026-10-28 17:42:44,641] INFO : Step 9300/50000 rank5 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1573 | Classification_binary: 0.3531 | Regression_cos_phi: 0.1969 | Regression_energy: 0.1613 | Regression_eta: 0.1547 | Regression_pt: 0.1717 | Regression_sin_phi: 0.2005 | LR: 9.25e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.8270s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank5 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank5 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2809 | Regression_cos_phi: 0.1573 | Regression_energy: 0.1373 | Regression_eta: 0.1643 | Regression_pt: 0.1416 | Regression_sin_phi: 0.1704 | LR: 9.24e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4117s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank5 [2026-10-28 17:51:11,693] INFO : Step 9500/50000 rank5 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0967 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1359 | Regression_energy: 0.1447 | Regression_eta: 0.1045 | Regression_pt: 0.1488 | Regression_sin_phi: 0.1217 | LR: 9.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.2529s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank5 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank5 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1070 | Classification_binary: 0.3547 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1488 | Regression_eta: 0.1373 | Regression_pt: 0.1533 | Regression_sin_phi: 0.1832 | LR: 9.21e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.8379s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank5 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank5 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.2662 | Regression_cos_phi: 0.1327 | Regression_energy: 0.1407 | Regression_eta: 0.1172 | Regression_pt: 0.1435 | Regression_sin_phi: 0.1153 | LR: 9.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9941s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank5 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank5 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0946 | Regression_energy: 0.1477 | Regression_eta: 0.0805 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1012 | LR: 9.17e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.6878s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank5 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank5 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1330 | Classification_binary: 0.3473 | Regression_cos_phi: 0.1724 | Regression_energy: 0.1467 | Regression_eta: 0.1496 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1581 | LR: 9.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1772s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank5 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank5 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1005 | Classification_binary: 0.3053 | Regression_cos_phi: 0.1968 | Regression_energy: 0.1499 | Regression_eta: 0.1117 | Regression_pt: 0.1568 | Regression_sin_phi: 0.1614 | LR: 9.14e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.1276s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank5 [2026-10-28 18:11:36,667] INFO : Running validation on rank5 +rank5 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:12:31,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:12:31,988] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 18:12:58,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:12:59,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:13:00,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:13:00,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:13:10,984] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 18:13:12,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:13:12,990] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 18:13:28,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:13:29,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:13:30,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:13:31,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:13:41,762] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:13:48,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:13:49,089] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 18:14:04,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:14:04,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:14:05,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:14:06,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:14:16,306] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 18:18:32,573] INFO : Step 10100/50000 rank5 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2733 | Regression_cos_phi: 0.1132 | Regression_energy: 0.1516 | Regression_eta: 0.1122 | Regression_pt: 0.1605 | Regression_sin_phi: 0.1003 | LR: 9.12e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.2345s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank5 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank5 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1006 | Classification_binary: 0.2964 | Regression_cos_phi: 0.1466 | Regression_energy: 0.1335 | Regression_eta: 0.1178 | Regression_pt: 0.1380 | Regression_sin_phi: 0.1212 | LR: 9.11e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5696s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank5 [2026-10-28 18:26:52,298] INFO : Step 10300/50000 rank5 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.3449 | Regression_cos_phi: 0.1193 | Regression_energy: 0.1354 | Regression_eta: 0.1089 | Regression_pt: 0.1411 | Regression_sin_phi: 0.1339 | LR: 9.09e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.3477s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank5 [2026-10-28 18:31:09,158] INFO : Step 10400/50000 rank5 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.2673 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1345 | Regression_eta: 0.1044 | Regression_pt: 0.1439 | Regression_sin_phi: 0.1378 | LR: 9.07e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9069s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank5 [2026-10-28 18:35:25,945] INFO : Step 10500/50000 rank5 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0852 | Classification_binary: 0.2638 | Regression_cos_phi: 0.0983 | Regression_energy: 0.1126 | Regression_eta: 0.0935 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0838 | LR: 9.06e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.8306s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank5 [2026-10-28 18:39:26,895] INFO : Step 10600/50000 rank5 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2954 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1231 | Regression_eta: 0.0896 | Regression_pt: 0.1279 | Regression_sin_phi: 0.1251 | LR: 9.04e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2175s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank5 [2026-10-28 18:43:27,489] INFO : Step 10700/50000 rank5 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.2982 | Regression_cos_phi: 0.1342 | Regression_energy: 0.1297 | Regression_eta: 0.0886 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1052 | LR: 9.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2756s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank5 [2026-10-28 18:47:34,478] INFO : Step 10800/50000 rank5 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0738 | Regression_energy: 0.1258 | Regression_eta: 0.0661 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0796 | LR: 9.00e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.3736s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank5 [2026-10-28 18:51:51,745] INFO : Step 10900/50000 rank5 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.2751 | Regression_cos_phi: 0.1344 | Regression_energy: 0.1366 | Regression_eta: 0.0910 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0959 | LR: 8.99e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6021s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank5 [2026-10-28 18:55:59,247] INFO : Step 11000/50000 rank5 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.3437 | Regression_cos_phi: 0.1249 | Regression_energy: 0.1154 | Regression_eta: 0.1033 | Regression_pt: 0.1199 | Regression_sin_phi: 0.1113 | LR: 8.97e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8646s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank5 [2026-10-28 18:56:00,217] INFO : Running validation on rank5 +rank5 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:56:54,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:56:55,520] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 18:57:21,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:57:22,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:57:23,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:57:24,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:57:34,639] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 18:57:35,272] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:57:35,835] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 18:57:52,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:57:53,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:57:54,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:57:55,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:58:05,781] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:58:06,769] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 18:58:22,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:58:23,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:58:23,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:58:24,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:58:34,891] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 19:03:04,100] INFO : Step 11100/50000 rank5 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2577 | Regression_cos_phi: 0.0762 | Regression_energy: 0.1194 | Regression_eta: 0.0699 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0767 | LR: 8.95e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.4750s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank5 [2026-10-28 19:07:10,329] INFO : Step 11200/50000 rank5 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2116 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1325 | Regression_eta: 0.1094 | Regression_pt: 0.1447 | Regression_sin_phi: 0.1497 | LR: 8.93e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8091s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank5 [2026-10-28 19:11:04,857] INFO : Step 11300/50000 rank5 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.3256 | Regression_cos_phi: 0.1281 | Regression_energy: 0.1228 | Regression_eta: 0.1193 | Regression_pt: 0.1292 | Regression_sin_phi: 0.1366 | LR: 8.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2487s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank5 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank5 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2761 | Regression_cos_phi: 0.1209 | Regression_energy: 0.1271 | Regression_eta: 0.0967 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0975 | LR: 8.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0019s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank5 [2026-10-28 19:18:58,942] INFO : Step 11500/50000 rank5 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2347 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1326 | Regression_eta: 0.0738 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0864 | LR: 8.88e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.4537s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank5 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank5 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0867 | Classification_binary: 0.2555 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1001 | Regression_eta: 0.0805 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0793 | LR: 8.86e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8328s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank5 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank5 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0941 | Classification_binary: 0.3081 | Regression_cos_phi: 0.1163 | Regression_energy: 0.1115 | Regression_eta: 0.1021 | Regression_pt: 0.1166 | Regression_sin_phi: 0.1386 | LR: 8.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1781s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank5 [2026-10-28 19:30:54,021] INFO : Step 11800/50000 rank5 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2247 | Regression_cos_phi: 0.1054 | Regression_energy: 0.1116 | Regression_eta: 0.0814 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0703 | LR: 8.82e-04 | DataLoad Time: 0.0241s | Model Forward Time: 6.0633s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank5 [2026-10-28 19:34:55,961] INFO : Step 11900/50000 rank5 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2476 | Regression_cos_phi: 0.1593 | Regression_energy: 0.1161 | Regression_eta: 0.1050 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1004 | LR: 8.80e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9522s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank5 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank5 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.3480 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1130 | Regression_eta: 0.1046 | Regression_pt: 0.1153 | Regression_sin_phi: 0.1022 | LR: 8.78e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3557s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank5 [2026-10-28 19:39:04,976] INFO : Running validation on rank5 +rank5 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 19:39:04,977] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 19:39:55,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:39:55,581] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 19:40:21,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 19:40:22,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 19:40:23,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 19:40:24,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 19:40:34,162] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 19:40:34,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:40:35,408] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 19:40:51,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 19:40:51,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 19:40:52,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 19:40:53,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 19:41:03,920] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:41:05,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:41:05,612] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 19:41:20,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 19:41:21,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 19:41:22,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 19:41:22,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 19:41:32,977] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 19:45:57,644] INFO : Step 12100/50000 rank5 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0935 | Regression_eta: 0.0839 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0823 | LR: 8.76e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8609s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank5 [2026-10-28 19:50:02,390] INFO : Step 12200/50000 rank5 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1141 | Classification_binary: 0.2568 | Regression_cos_phi: 0.1053 | Regression_energy: 0.1044 | Regression_eta: 0.0928 | Regression_pt: 0.1088 | Regression_sin_phi: 0.1067 | LR: 8.74e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.9610s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank5 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank5 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1034 | Classification_binary: 0.2904 | Regression_cos_phi: 0.0980 | Regression_energy: 0.1087 | Regression_eta: 0.0966 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0984 | LR: 8.72e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4610s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank5 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank5 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.2788 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1250 | Regression_eta: 0.0989 | Regression_pt: 0.1294 | Regression_sin_phi: 0.1004 | LR: 8.70e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9473s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank5 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank5 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1864 | Regression_cos_phi: 0.1157 | Regression_energy: 0.0955 | Regression_eta: 0.0568 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0640 | LR: 8.68e-04 | DataLoad Time: 0.0238s | Model Forward Time: 4.6930s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank5 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank5 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.2782 | Regression_cos_phi: 0.0999 | Regression_energy: 0.1305 | Regression_eta: 0.0993 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0959 | LR: 8.66e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.9467s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank5 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank5 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2890 | Regression_cos_phi: 0.0863 | Regression_energy: 0.1137 | Regression_eta: 0.0735 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0967 | LR: 8.64e-04 | DataLoad Time: 0.0111s | Model Forward Time: 5.2292s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank5 [2026-10-28 20:14:52,958] INFO : Step 12800/50000 rank5 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2441 | Regression_cos_phi: 0.0916 | Regression_energy: 0.1057 | Regression_eta: 0.0931 | Regression_pt: 0.1121 | Regression_sin_phi: 0.1226 | LR: 8.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8553s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank5 [2026-10-28 20:19:01,949] INFO : Step 12900/50000 rank5 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2667 | Regression_cos_phi: 0.0902 | Regression_energy: 0.1014 | Regression_eta: 0.0872 | Regression_pt: 0.1034 | Regression_sin_phi: 0.1222 | LR: 8.60e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.5188s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank5 [2026-10-28 20:23:25,507] INFO : Step 13000/50000 rank5 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2961 | Regression_cos_phi: 0.1298 | Regression_energy: 0.1057 | Regression_eta: 0.0972 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0999 | LR: 8.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 10.3374s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank5 [2026-10-28 20:23:26,442] INFO : Running validation on rank5 +rank5 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 20:24:18,600] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 20:24:45,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 20:24:45,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 20:24:46,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 20:24:47,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 20:24:57,924] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:25:04,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:25:04,757] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 20:25:20,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 20:25:21,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 20:25:22,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 20:25:23,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 20:25:33,833] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:25:34,725] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 20:25:50,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 20:25:50,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 20:25:51,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 20:25:52,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 20:26:02,183] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 20:30:28,057] INFO : Step 13100/50000 rank5 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0677 | Regression_energy: 0.1121 | Regression_eta: 0.0665 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0833 | LR: 8.56e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.0197s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank5 [2026-10-28 20:34:38,790] INFO : Step 13200/50000 rank5 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.2670 | Regression_cos_phi: 0.0811 | Regression_energy: 0.1019 | Regression_eta: 0.0939 | Regression_pt: 0.1053 | Regression_sin_phi: 0.1058 | LR: 8.54e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0418s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank5 [2026-10-28 20:38:50,942] INFO : Step 13300/50000 rank5 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.2770 | Regression_cos_phi: 0.1223 | Regression_energy: 0.1071 | Regression_eta: 0.0864 | Regression_pt: 0.1109 | Regression_sin_phi: 0.1059 | LR: 8.52e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.4737s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank5 [2026-10-28 20:43:04,349] INFO : Step 13400/50000 rank5 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.3114 | Regression_cos_phi: 0.0963 | Regression_energy: 0.1036 | Regression_eta: 0.0883 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0907 | LR: 8.50e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2844s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank5 [2026-10-28 20:47:13,647] INFO : Step 13500/50000 rank5 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1949 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1075 | Regression_eta: 0.0967 | Regression_pt: 0.1110 | Regression_sin_phi: 0.1229 | LR: 8.48e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.1196s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank5 [2026-10-28 20:51:26,251] INFO : Step 13600/50000 rank5 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2371 | Regression_cos_phi: 0.0741 | Regression_energy: 0.1069 | Regression_eta: 0.0723 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0693 | LR: 8.45e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8035s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank5 [2026-10-28 20:55:41,373] INFO : Step 13700/50000 rank5 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.3142 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0847 | Regression_eta: 0.0730 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0824 | LR: 8.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8081s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank5 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank5 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0657 | Regression_energy: 0.1061 | Regression_eta: 0.0537 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0749 | LR: 8.41e-04 | DataLoad Time: 0.0224s | Model Forward Time: 6.1137s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank5 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank5 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.2315 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0956 | Regression_eta: 0.0760 | Regression_pt: 0.0973 | Regression_sin_phi: 0.1070 | LR: 8.39e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.1585s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank5 [2026-10-28 21:08:24,062] INFO : Step 14000/50000 rank5 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1022 | Classification_binary: 0.3125 | Regression_cos_phi: 0.1323 | Regression_energy: 0.0932 | Regression_eta: 0.1100 | Regression_pt: 0.0982 | Regression_sin_phi: 0.1559 | LR: 8.37e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.9196s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank5 [2026-10-28 21:08:24,998] INFO : Running validation on rank5 +rank5 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:09:16,436] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 21:09:43,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:09:44,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:09:44,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:09:45,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:09:55,927] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 21:09:56,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:09:56,997] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 21:10:13,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:10:14,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:10:15,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:10:16,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:10:26,297] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:10:26,833] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 21:10:42,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:10:43,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:10:44,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:10:44,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:10:54,980] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank5 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0909 | Classification_binary: 0.2875 | Regression_cos_phi: 0.1092 | Regression_energy: 0.0889 | Regression_eta: 0.0794 | Regression_pt: 0.0916 | Regression_sin_phi: 0.1069 | LR: 8.35e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5888s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank5 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank5 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1844 | Regression_cos_phi: 0.1125 | Regression_energy: 0.0825 | Regression_eta: 0.0771 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0924 | LR: 8.32e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4846s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank5 [2026-10-28 21:24:03,128] INFO : Step 14300/50000 rank5 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0914 | Classification_binary: 0.2595 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0958 | Regression_eta: 0.0889 | Regression_pt: 0.1002 | Regression_sin_phi: 0.1125 | LR: 8.30e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0224s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank5 [2026-10-28 21:28:09,727] INFO : Step 14400/50000 rank5 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.3031 | Regression_cos_phi: 0.1365 | Regression_energy: 0.0978 | Regression_eta: 0.0916 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0933 | LR: 8.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0257s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank5 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank5 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0843 | Regression_eta: 0.0525 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0724 | LR: 8.26e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.3473s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank5 [2026-10-28 21:36:11,051] INFO : Step 14600/50000 rank5 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0927 | Regression_eta: 0.0851 | Regression_pt: 0.0983 | Regression_sin_phi: 0.1170 | LR: 8.24e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.9768s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank5 [2026-10-28 21:40:16,608] INFO : Step 14700/50000 rank5 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2750 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0897 | Regression_eta: 0.1008 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0792 | LR: 8.21e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.1358s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank5 [2026-10-28 21:44:33,909] INFO : Step 14800/50000 rank5 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1967 | Regression_cos_phi: 0.1092 | Regression_energy: 0.0996 | Regression_eta: 0.0580 | Regression_pt: 0.1031 | Regression_sin_phi: 0.1087 | LR: 8.19e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3492s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank5 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank5 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0861 | Regression_eta: 0.0546 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0532 | LR: 8.17e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.3232s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank5 [2026-10-28 21:52:53,291] INFO : Step 15000/50000 rank5 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0933 | Regression_eta: 0.0696 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0821 | LR: 8.15e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7242s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank5 [2026-10-28 21:52:54,210] INFO : Running validation on rank5 +rank5 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:53:44,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:53:44,570] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 21:54:10,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:54:11,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:54:12,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:54:13,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:54:23,634] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:54:24,123] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:54:24,599] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 21:54:40,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:54:41,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:54:42,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:54:43,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:54:53,572] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:54:54,913] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 21:55:10,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:55:11,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:55:12,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:55:12,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:55:22,974] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 21:59:40,818] INFO : Step 15100/50000 rank5 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2539 | Regression_cos_phi: 0.0917 | Regression_energy: 0.1015 | Regression_eta: 0.0679 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0800 | LR: 8.12e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.6876s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank5 [2026-10-28 22:03:40,545] INFO : Step 15200/50000 rank5 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0712 | Regression_energy: 0.1075 | Regression_eta: 0.0583 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0680 | LR: 8.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9626s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank5 [2026-10-28 22:07:35,577] INFO : Step 15300/50000 rank5 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2436 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0768 | Regression_eta: 0.0743 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0703 | LR: 8.08e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8185s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank5 [2026-10-28 22:11:47,342] INFO : Step 15400/50000 rank5 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2895 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0859 | Regression_eta: 0.1022 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0841 | LR: 8.05e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1734s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank5 [2026-10-28 22:16:06,325] INFO : Step 15500/50000 rank5 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0768 | Regression_eta: 0.0824 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0786 | LR: 8.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9217s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank5 [2026-10-28 22:20:18,326] INFO : Step 15600/50000 rank5 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.2111 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0792 | Regression_eta: 0.0791 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0951 | LR: 8.01e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4208s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank5 [2026-10-28 22:24:30,251] INFO : Step 15700/50000 rank5 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.3067 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0867 | Regression_eta: 0.0744 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0847 | LR: 7.98e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.9299s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank5 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank5 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2587 | Regression_cos_phi: 0.1229 | Regression_energy: 0.1102 | Regression_eta: 0.0993 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0933 | LR: 7.96e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3023s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank5 [2026-10-28 22:32:57,712] INFO : Step 15900/50000 rank5 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2192 | Regression_cos_phi: 0.0840 | Regression_energy: 0.1096 | Regression_eta: 0.1025 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0830 | LR: 7.94e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.3910s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank5 [2026-10-28 22:37:10,742] INFO : Step 16000/50000 rank5 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2327 | Regression_cos_phi: 0.1126 | Regression_energy: 0.0897 | Regression_eta: 0.0828 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0941 | LR: 7.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0613s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank5 [2026-10-28 22:37:11,681] INFO : Running validation on rank5 +rank5 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 22:38:03,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:38:04,032] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 22:38:30,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 22:38:31,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 22:38:32,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 22:38:33,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 22:38:43,054] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 22:38:43,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:38:44,321] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 22:39:01,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 22:39:02,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 22:39:03,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 22:39:04,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 22:39:14,410] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 22:39:15,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:39:16,104] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 22:39:31,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 22:39:32,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 22:39:33,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 22:39:33,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 22:39:43,734] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 22:44:03,866] INFO : Step 16100/50000 rank5 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1150 | Regression_energy: 0.0989 | Regression_eta: 0.1082 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0993 | LR: 7.89e-04 | DataLoad Time: 0.0124s | Model Forward Time: 4.7746s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank5 [2026-10-28 22:48:09,286] INFO : Step 16200/50000 rank5 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2118 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0892 | Regression_eta: 0.0978 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0749 | LR: 7.86e-04 | DataLoad Time: 0.0226s | Model Forward Time: 3.6645s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank5 [2026-10-28 22:52:16,253] INFO : Step 16300/50000 rank5 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0749 | Regression_eta: 0.0741 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0953 | LR: 7.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1373s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank5 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank5 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2289 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0777 | Regression_eta: 0.0616 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0694 | LR: 7.81e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4170s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank5 [2026-10-28 23:00:23,724] INFO : Step 16500/50000 rank5 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0737 | Regression_eta: 0.0740 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0732 | LR: 7.79e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9994s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank5 [2026-10-28 23:04:26,317] INFO : Step 16600/50000 rank5 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0802 | Regression_eta: 0.0533 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0617 | LR: 7.77e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.3622s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank5 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank5 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2673 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0867 | Regression_eta: 0.0728 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0685 | LR: 7.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0574s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank5 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank5 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2127 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0776 | Regression_eta: 0.0809 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0868 | LR: 7.72e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.0650s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank5 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank5 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0924 | Regression_eta: 0.0375 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0540 | LR: 7.69e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7700s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank5 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank5 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2558 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0737 | Regression_eta: 0.0926 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0745 | LR: 7.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9356s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank5 [2026-10-28 23:20:37,935] INFO : Running validation on rank5 +rank5 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 23:21:28,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:21:29,098] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 23:21:56,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 23:21:58,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 23:22:00,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 23:22:02,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 23:22:12,095] INFO : Time taken to make predictions on device 5 is: 0.32 min +rank5 [2026-10-28 23:22:12,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:22:12,959] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 23:22:29,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 23:22:30,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 23:22:30,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 23:22:31,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 23:22:41,920] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:22:42,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:22:42,879] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 23:22:58,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 23:22:59,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 23:22:59,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 23:23:00,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 23:23:10,781] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 23:27:38,314] INFO : Step 17100/50000 rank5 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0821 | Regression_eta: 0.0557 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0783 | LR: 7.64e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.1670s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank5 [2026-10-28 23:31:38,672] INFO : Step 17200/50000 rank5 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0853 | Regression_eta: 0.0563 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0567 | LR: 7.62e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.1968s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank5 [2026-10-28 23:35:49,547] INFO : Step 17300/50000 rank5 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0777 | Regression_eta: 0.0602 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0625 | LR: 7.59e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.5191s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank5 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank5 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0789 | Regression_eta: 0.0647 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0828 | LR: 7.57e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.3517s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank5 [2026-10-28 23:43:53,975] INFO : Step 17500/50000 rank5 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2176 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0763 | Regression_eta: 0.0564 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0724 | LR: 7.54e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9636s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank5 [2026-10-28 23:48:08,874] INFO : Step 17600/50000 rank5 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0755 | Regression_eta: 0.1130 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0559 | LR: 7.52e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5562s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank5 [2026-10-28 23:52:17,700] INFO : Step 17700/50000 rank5 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2612 | Regression_cos_phi: 0.1200 | Regression_energy: 0.0853 | Regression_eta: 0.0744 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0690 | LR: 7.49e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8663s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank5 [2026-10-28 23:56:40,920] INFO : Step 17800/50000 rank5 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2703 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0715 | Regression_eta: 0.0835 | Regression_pt: 0.0750 | Regression_sin_phi: 0.1134 | LR: 7.47e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1269s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank5 [2026-10-29 00:00:47,129] INFO : Step 17900/50000 rank5 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0748 | Regression_eta: 0.0620 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0570 | LR: 7.44e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2715s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank5 [2026-10-29 00:04:53,688] INFO : Step 18000/50000 rank5 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0844 | Regression_eta: 0.1019 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0873 | LR: 7.42e-04 | DataLoad Time: 0.0166s | Model Forward Time: 13.5871s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank5 [2026-10-29 00:04:54,634] INFO : Running validation on rank5 +rank5 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 00:04:54,634] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:05:43,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:05:43,820] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 00:06:10,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:06:11,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:06:12,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:06:13,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:06:23,691] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:06:24,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 00:06:40,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:06:41,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:06:42,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:06:43,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:06:53,807] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:06:55,432] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 00:07:11,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:07:11,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:07:12,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:07:13,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:07:23,066] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 00:11:38,772] INFO : Step 18100/50000 rank5 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2378 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0835 | Regression_eta: 0.0630 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0808 | LR: 7.39e-04 | DataLoad Time: 0.0267s | Model Forward Time: 4.2903s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank5 [2026-10-29 00:15:47,603] INFO : Step 18200/50000 rank5 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0828 | Regression_eta: 0.0641 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0741 | LR: 7.36e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1155s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank5 [2026-10-29 00:19:54,087] INFO : Step 18300/50000 rank5 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0744 | Regression_eta: 0.0820 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0925 | LR: 7.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8062s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank5 [2026-10-29 00:24:07,390] INFO : Step 18400/50000 rank5 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2293 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0685 | Regression_eta: 0.0772 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0891 | LR: 7.31e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4811s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank5 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank5 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0891 | Regression_eta: 0.0687 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0820 | LR: 7.29e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5451s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank5 [2026-10-29 00:31:24,472] INFO : Step 18600/50000 rank5 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0964 | Regression_eta: 0.0608 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0789 | LR: 7.26e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1903s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank5 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank5 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2281 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0839 | Regression_eta: 0.0577 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0824 | LR: 7.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.9493s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank5 [2026-10-29 00:38:49,580] INFO : Step 18800/50000 rank5 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0779 | Regression_eta: 0.1349 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0723 | LR: 7.21e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4365s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank5 [2026-10-29 00:42:28,237] INFO : Step 18900/50000 rank5 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0830 | Regression_eta: 0.0500 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0547 | LR: 7.18e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0111s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank5 [2026-10-29 00:46:10,109] INFO : Step 19000/50000 rank5 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2259 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0909 | Regression_eta: 0.0701 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0958 | LR: 7.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1523s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank5 [2026-10-29 00:46:11,054] INFO : Running validation on rank5 +rank5 [2026-10-29 00:46:11,055] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 00:46:11,055] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:46:58,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:46:59,171] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 00:47:24,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:47:25,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:47:26,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:47:27,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:47:37,612] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 00:47:38,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:47:38,900] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 00:47:54,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:47:55,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:47:56,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:47:57,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:48:07,790] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:48:09,177] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 00:48:24,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:48:25,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:48:25,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:48:26,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:48:36,689] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank5 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2544 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0783 | Regression_eta: 0.0712 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0746 | LR: 7.13e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5908s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank5 [2026-10-29 00:56:31,489] INFO : Step 19200/50000 rank5 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0687 | Regression_eta: 0.0474 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0664 | LR: 7.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7715s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank5 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank5 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0984 | Regression_eta: 0.0474 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0434 | LR: 7.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6459s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank5 [2026-10-29 01:03:57,849] INFO : Step 19400/50000 rank5 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0795 | Regression_energy: 0.1023 | Regression_eta: 0.0609 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0685 | LR: 7.05e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank5 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank5 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2539 | Regression_cos_phi: 0.1056 | Regression_energy: 0.0882 | Regression_eta: 0.0820 | Regression_pt: 0.0930 | Regression_sin_phi: 0.1033 | LR: 7.02e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0510s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank5 [2026-10-29 01:11:25,842] INFO : Step 19600/50000 rank5 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0852 | Regression_eta: 0.0538 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0524 | LR: 7.00e-04 | DataLoad Time: 0.0226s | Model Forward Time: 3.9370s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank5 [2026-10-29 01:15:28,503] INFO : Step 19700/50000 rank5 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0786 | Classification_binary: 0.2217 | Regression_cos_phi: 0.1214 | Regression_energy: 0.0859 | Regression_eta: 0.0992 | Regression_pt: 0.0891 | Regression_sin_phi: 0.1207 | LR: 6.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.3555s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank5 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank5 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2698 | Regression_cos_phi: 0.0905 | Regression_energy: 0.1052 | Regression_eta: 0.0786 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1072 | LR: 6.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3396s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank5 [2026-10-29 01:23:26,314] INFO : Step 19900/50000 rank5 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2126 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1081 | Regression_eta: 0.0781 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0819 | LR: 6.92e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7481s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank5 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank5 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0952 | Regression_eta: 0.0863 | Regression_pt: 0.0993 | Regression_sin_phi: 0.1011 | LR: 6.89e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8752s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank5 [2026-10-29 01:27:16,848] INFO : Running validation on rank5 +rank5 [2026-10-29 01:27:16,849] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 01:27:16,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 01:28:06,613] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:28:07,182] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 01:28:33,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 01:28:34,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 01:28:35,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 01:28:36,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 01:28:46,919] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:28:48,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:28:48,586] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 01:29:04,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 01:29:05,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 01:29:06,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 01:29:07,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 01:29:17,454] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 01:29:18,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:29:18,549] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 01:29:34,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 01:29:34,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 01:29:35,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 01:29:36,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 01:29:46,085] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 01:33:54,475] INFO : Step 20100/50000 rank5 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2507 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0843 | Regression_eta: 0.0763 | Regression_pt: 0.0901 | Regression_sin_phi: 0.1105 | LR: 6.86e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.4716s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank5 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank5 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0782 | Regression_eta: 0.0565 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0770 | LR: 6.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9218s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank5 [2026-10-29 01:41:55,632] INFO : Step 20300/50000 rank5 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0706 | Regression_eta: 0.0667 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0700 | LR: 6.81e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6977s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank5 [2026-10-29 01:45:46,289] INFO : Step 20400/50000 rank5 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2451 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0854 | Regression_eta: 0.0667 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0791 | LR: 6.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1288s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank5 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank5 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2338 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0668 | Regression_eta: 0.0861 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0801 | LR: 6.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3411s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank5 [2026-10-29 01:53:32,128] INFO : Step 20600/50000 rank5 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.1960 | Regression_cos_phi: 0.0964 | Regression_energy: 0.1002 | Regression_eta: 0.0674 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0954 | LR: 6.73e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.0685s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank5 [2026-10-29 01:57:29,941] INFO : Step 20700/50000 rank5 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0822 | Regression_eta: 0.0728 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0711 | LR: 6.70e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.6607s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank5 [2026-10-29 02:01:35,425] INFO : Step 20800/50000 rank5 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2569 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0875 | Regression_eta: 0.0633 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0795 | LR: 6.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8004s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank5 [2026-10-29 02:05:49,595] INFO : Step 20900/50000 rank5 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0923 | Regression_eta: 0.0725 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0640 | LR: 6.65e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.1302s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank5 [2026-10-29 02:09:53,644] INFO : Step 21000/50000 rank5 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1790 | Regression_cos_phi: 0.1020 | Regression_energy: 0.0959 | Regression_eta: 0.0666 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0816 | LR: 6.62e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.8427s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank5 [2026-10-29 02:09:54,611] INFO : Running validation on rank5 +rank5 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:09:54,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:10:45,388] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:10:45,900] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 02:11:12,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:11:13,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:11:14,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:11:15,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:11:25,503] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:11:25,729] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:11:26,308] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 02:11:43,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:11:44,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:11:45,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:11:47,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:11:57,117] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 02:11:57,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:11:57,809] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 02:12:13,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:12:14,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:12:14,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:12:15,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:12:25,545] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 02:16:43,091] INFO : Step 21100/50000 rank5 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2548 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0973 | Regression_eta: 0.0853 | Regression_pt: 0.1025 | Regression_sin_phi: 0.1004 | LR: 6.59e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5463s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank5 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank5 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank5 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0781 | Regression_eta: 0.0527 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0703 | LR: 6.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1644s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank5 [2026-10-29 02:24:40,427] INFO : Step 21300/50000 rank5 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0933 | Regression_eta: 0.0594 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0781 | LR: 6.54e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1546s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank5 [2026-10-29 02:28:35,146] INFO : Step 21400/50000 rank5 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2405 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0810 | Regression_eta: 0.0627 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0788 | LR: 6.51e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4151s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank5 [2026-10-29 02:32:39,449] INFO : Step 21500/50000 rank5 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1984 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0873 | Regression_eta: 0.0714 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0689 | LR: 6.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7862s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank5 [2026-10-29 02:36:35,118] INFO : Step 21600/50000 rank5 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0888 | Regression_eta: 0.0811 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0972 | LR: 6.45e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.8430s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank5 [2026-10-29 02:40:30,747] INFO : Step 21700/50000 rank5 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2147 | Regression_cos_phi: 0.0653 | Regression_energy: 0.1069 | Regression_eta: 0.0525 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0730 | LR: 6.43e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.5968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank5 [2026-10-29 02:44:20,671] INFO : Step 21800/50000 rank5 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0922 | Regression_eta: 0.0820 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0771 | LR: 6.40e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.1211s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank5 [2026-10-29 02:48:12,872] INFO : Step 21900/50000 rank5 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.1855 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0901 | Regression_eta: 0.0872 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0814 | LR: 6.37e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2340s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank5 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank5 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2786 | Regression_cos_phi: 0.1256 | Regression_energy: 0.0921 | Regression_eta: 0.0896 | Regression_pt: 0.0959 | Regression_sin_phi: 0.1163 | LR: 6.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1560s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank5 [2026-10-29 02:52:09,891] INFO : Running validation on rank5 +rank5 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:52:57,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:52:58,579] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 02:53:26,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:53:27,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:53:28,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:53:29,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:53:39,610] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:53:40,600] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 02:53:56,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:53:57,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:53:58,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:53:59,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:54:09,184] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:54:10,683] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 02:54:25,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:54:26,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:54:27,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:54:27,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:54:37,886] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank5 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2860 | Regression_cos_phi: 0.1263 | Regression_energy: 0.1020 | Regression_eta: 0.0972 | Regression_pt: 0.1063 | Regression_sin_phi: 0.1246 | LR: 6.32e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4346s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank5 [2026-10-29 03:02:36,687] INFO : Step 22200/50000 rank5 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0719 | Regression_energy: 0.1039 | Regression_eta: 0.0661 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0922 | LR: 6.29e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6560s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank5 [2026-10-29 03:06:28,834] INFO : Step 22300/50000 rank5 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2569 | Regression_cos_phi: 0.1143 | Regression_energy: 0.1050 | Regression_eta: 0.0920 | Regression_pt: 0.1078 | Regression_sin_phi: 0.1154 | LR: 6.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8521s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank5 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank5 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2738 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1060 | Regression_eta: 0.0935 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0996 | LR: 6.23e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.5928s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank5 [2026-10-29 03:14:16,070] INFO : Step 22500/50000 rank5 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2349 | Regression_cos_phi: 0.0847 | Regression_energy: 0.1039 | Regression_eta: 0.0936 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0959 | LR: 6.20e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8997s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank5 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank5 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2052 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0994 | Regression_eta: 0.0759 | Regression_pt: 0.1034 | Regression_sin_phi: 0.1023 | LR: 6.18e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5793s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank5 [2026-10-29 03:22:04,589] INFO : Step 22700/50000 rank5 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0852 | Classification_binary: 0.2618 | Regression_cos_phi: 0.1372 | Regression_energy: 0.1010 | Regression_eta: 0.0725 | Regression_pt: 0.1056 | Regression_sin_phi: 0.1003 | LR: 6.15e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3260s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank5 [2026-10-29 03:26:06,869] INFO : Step 22800/50000 rank5 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0731 | Regression_energy: 0.1218 | Regression_eta: 0.0710 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0818 | LR: 6.12e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2790s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank5 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank5 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0706 | Regression_energy: 0.1065 | Regression_eta: 0.0570 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0670 | LR: 6.09e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6097s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank5 [2026-10-29 03:34:03,730] INFO : Step 23000/50000 rank5 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0736 | Regression_energy: 0.1242 | Regression_eta: 0.0831 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0922 | LR: 6.06e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6399s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank5 [2026-10-29 03:34:04,743] INFO : Running validation on rank5 +rank5 [2026-10-29 03:34:04,744] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 03:34:04,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 03:34:52,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:34:53,014] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 03:35:18,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 03:35:19,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 03:35:20,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 03:35:21,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 03:35:31,657] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:35:32,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 03:35:49,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 03:35:50,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 03:35:50,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 03:35:51,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 03:36:01,937] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 03:36:02,378] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:36:02,911] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 03:36:18,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 03:36:18,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 03:36:19,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 03:36:20,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 03:36:30,307] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 03:40:33,162] INFO : Step 23100/50000 rank5 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1076 | Regression_energy: 0.0980 | Regression_eta: 0.1098 | Regression_pt: 0.1019 | Regression_sin_phi: 0.1075 | LR: 6.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4149s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank5 [2026-10-29 03:44:14,417] INFO : Step 23200/50000 rank5 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0831 | Regression_eta: 0.0735 | Regression_pt: 0.0880 | Regression_sin_phi: 0.1068 | LR: 6.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4837s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank5 [2026-10-29 03:47:55,310] INFO : Step 23300/50000 rank5 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2684 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1117 | Regression_eta: 0.1114 | Regression_pt: 0.1164 | Regression_sin_phi: 0.1059 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9646s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank5 [2026-10-29 03:51:33,797] INFO : Step 23400/50000 rank5 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0766 | Regression_energy: 0.1077 | Regression_eta: 0.0749 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0823 | LR: 5.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1976s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank5 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank5 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.2134 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0934 | Regression_eta: 0.0736 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0861 | LR: 5.92e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0139s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank5 [2026-10-29 03:59:15,166] INFO : Step 23600/50000 rank5 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0909 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1030 | Regression_energy: 0.1068 | Regression_eta: 0.0629 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0883 | LR: 5.90e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.9372s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank5 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank5 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2538 | Regression_cos_phi: 0.0944 | Regression_energy: 0.1041 | Regression_eta: 0.0851 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0954 | LR: 5.87e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3842s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank5 [2026-10-29 04:06:50,323] INFO : Step 23800/50000 rank5 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2228 | Regression_cos_phi: 0.1033 | Regression_energy: 0.0969 | Regression_eta: 0.1028 | Regression_pt: 0.1019 | Regression_sin_phi: 0.1202 | LR: 5.84e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.0045s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank5 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank5 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0540 | Regression_energy: 0.1107 | Regression_eta: 0.0609 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0873 | LR: 5.81e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3355s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank5 [2026-10-29 04:14:57,721] INFO : Step 24000/50000 rank5 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2347 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0962 | Regression_eta: 0.0709 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0960 | LR: 5.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1437s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank5 [2026-10-29 04:14:58,739] INFO : Running validation on rank5 +rank5 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:14:58,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:15:47,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:15:48,539] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 04:16:15,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:16:16,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:16:17,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:16:18,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:16:28,254] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:16:28,932] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:16:29,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 04:16:45,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:16:46,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:16:47,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:16:48,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:16:58,897] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:16:59,582] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 04:17:15,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:17:15,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:17:16,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:17:17,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:17:27,087] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 04:21:44,037] INFO : Step 24100/50000 rank5 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2602 | Regression_cos_phi: 0.0946 | Regression_energy: 0.1113 | Regression_eta: 0.0663 | Regression_pt: 0.1167 | Regression_sin_phi: 0.1073 | LR: 5.75e-04 | DataLoad Time: 0.0196s | Model Forward Time: 7.7524s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank5 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank5 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2226 | Regression_cos_phi: 0.1008 | Regression_energy: 0.1020 | Regression_eta: 0.0681 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0720 | LR: 5.73e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2953s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank5 [2026-10-29 04:29:22,270] INFO : Step 24300/50000 rank5 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0897 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0840 | Regression_energy: 0.1024 | Regression_eta: 0.0861 | Regression_pt: 0.1071 | Regression_sin_phi: 0.1005 | LR: 5.70e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0411s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank5 [2026-10-29 04:33:07,495] INFO : Step 24400/50000 rank5 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1027 | Classification_binary: 0.3068 | Regression_cos_phi: 0.1359 | Regression_energy: 0.1623 | Regression_eta: 0.1281 | Regression_pt: 0.1703 | Regression_sin_phi: 0.1590 | LR: 5.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2374s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank5 [2026-10-29 04:36:57,190] INFO : Step 24500/50000 rank5 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.3033 | Regression_cos_phi: 0.1197 | Regression_energy: 0.1133 | Regression_eta: 0.1128 | Regression_pt: 0.1186 | Regression_sin_phi: 0.1240 | LR: 5.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3253s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank5 [2026-10-29 04:40:40,560] INFO : Step 24600/50000 rank5 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0456 | Regression_energy: 0.1102 | Regression_eta: 0.0615 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0735 | LR: 5.61e-04 | DataLoad Time: 0.0264s | Model Forward Time: 3.9877s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank5 [2026-10-29 04:44:29,989] INFO : Step 24700/50000 rank5 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1185 | Regression_energy: 0.1187 | Regression_eta: 0.1198 | Regression_pt: 0.1264 | Regression_sin_phi: 0.1469 | LR: 5.58e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5086s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank5 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank5 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2681 | Regression_cos_phi: 0.0923 | Regression_energy: 0.1135 | Regression_eta: 0.1137 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0911 | LR: 5.56e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8642s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank5 [2026-10-29 04:52:00,597] INFO : Step 24900/50000 rank5 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0978 | Regression_energy: 0.1165 | Regression_eta: 0.0787 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0972 | LR: 5.53e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.9773s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank5 [2026-10-29 04:55:44,333] INFO : Step 25000/50000 rank5 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.2243 | Regression_cos_phi: 0.1173 | Regression_energy: 0.1229 | Regression_eta: 0.0773 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0978 | LR: 5.50e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2485s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank5 [2026-10-29 04:55:45,305] INFO : Running validation on rank5 +rank5 [2026-10-29 04:55:45,305] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:55:45,305] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:56:33,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:56:34,566] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 04:57:00,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:57:01,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:57:01,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:57:02,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:57:12,902] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 04:57:13,409] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:57:13,950] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 04:57:29,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:57:30,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:57:31,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:57:32,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:57:42,600] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:57:44,417] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 04:57:59,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:58:00,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:58:00,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:58:01,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:58:11,586] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 05:02:15,911] INFO : Step 25100/50000 rank5 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2616 | Regression_cos_phi: 0.0912 | Regression_energy: 0.1171 | Regression_eta: 0.0937 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0839 | LR: 5.47e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7066s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank5 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank5 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0938 | Regression_eta: 0.0682 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0729 | LR: 5.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2819s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank5 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank5 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1992 | Regression_cos_phi: 0.1090 | Regression_energy: 0.1118 | Regression_eta: 0.0921 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0815 | LR: 5.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8566s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank5 [2026-10-29 05:13:35,656] INFO : Step 25400/50000 rank5 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1316 | Regression_energy: 0.1251 | Regression_eta: 0.0859 | Regression_pt: 0.1325 | Regression_sin_phi: 0.1975 | LR: 5.39e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0677s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank5 [2026-10-29 05:17:27,534] INFO : Step 25500/50000 rank5 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2596 | Regression_cos_phi: 0.1024 | Regression_energy: 0.1043 | Regression_eta: 0.0727 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0779 | LR: 5.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0764s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank5 [2026-10-29 05:21:16,672] INFO : Step 25600/50000 rank5 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2132 | Regression_cos_phi: 0.1128 | Regression_energy: 0.1280 | Regression_eta: 0.1002 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0811 | LR: 5.33e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8755s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank5 [2026-10-29 05:25:00,204] INFO : Step 25700/50000 rank5 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2784 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1247 | Regression_eta: 0.0727 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1034 | LR: 5.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8872s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank5 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank5 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.3406 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1314 | Regression_eta: 0.0928 | Regression_pt: 0.1371 | Regression_sin_phi: 0.1134 | LR: 5.27e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5189s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank5 [2026-10-29 05:32:33,909] INFO : Step 25900/50000 rank5 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1902 | Regression_cos_phi: 0.1111 | Regression_energy: 0.1058 | Regression_eta: 0.0684 | Regression_pt: 0.1100 | Regression_sin_phi: 0.1127 | LR: 5.25e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.8728s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank5 [2026-10-29 05:36:28,636] INFO : Step 26000/50000 rank5 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2162 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1220 | Regression_eta: 0.0937 | Regression_pt: 0.1276 | Regression_sin_phi: 0.1404 | LR: 5.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 9.2339s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank5 [2026-10-29 05:36:29,553] INFO : Running validation on rank5 +rank5 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 05:37:19,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:37:20,020] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 05:37:45,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 05:37:46,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 05:37:46,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 05:37:47,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 05:37:57,882] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 05:37:58,606] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 05:38:15,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 05:38:16,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 05:38:17,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 05:38:18,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 05:38:28,030] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:38:28,895] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 05:38:44,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 05:38:44,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 05:38:45,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 05:38:46,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 05:38:56,386] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 05:43:04,442] INFO : Step 26100/50000 rank5 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0912 | Regression_energy: 0.1197 | Regression_eta: 0.0695 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0943 | LR: 5.19e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8573s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank5 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank5 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2345 | Regression_cos_phi: 0.1060 | Regression_energy: 0.0986 | Regression_eta: 0.0618 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0771 | LR: 5.16e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2712s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank5 [2026-10-29 05:50:52,484] INFO : Step 26300/50000 rank5 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0999 | Regression_eta: 0.0672 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0699 | LR: 5.13e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3831s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank5 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank5 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.3079 | Regression_cos_phi: 0.1599 | Regression_energy: 0.1418 | Regression_eta: 0.1316 | Regression_pt: 0.1490 | Regression_sin_phi: 0.1358 | LR: 5.10e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3822s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank5 [2026-10-29 05:58:42,122] INFO : Step 26500/50000 rank5 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.2754 | Regression_cos_phi: 0.0942 | Regression_energy: 0.1272 | Regression_eta: 0.0600 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0825 | LR: 5.08e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4055s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank5 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank5 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0848 | Regression_energy: 0.1180 | Regression_eta: 0.0625 | Regression_pt: 0.1227 | Regression_sin_phi: 0.1199 | LR: 5.05e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7666s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank5 [2026-10-29 06:06:39,600] INFO : Step 26700/50000 rank5 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.2197 | Regression_cos_phi: 0.1178 | Regression_energy: 0.1371 | Regression_eta: 0.0802 | Regression_pt: 0.1445 | Regression_sin_phi: 0.1423 | LR: 5.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.6701s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank5 [2026-10-29 06:10:41,254] INFO : Step 26800/50000 rank5 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.3199 | Regression_cos_phi: 0.1398 | Regression_energy: 0.1336 | Regression_eta: 0.1037 | Regression_pt: 0.1402 | Regression_sin_phi: 0.1123 | LR: 4.99e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2405s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank5 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank5 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2476 | Regression_cos_phi: 0.0887 | Regression_energy: 0.1002 | Regression_eta: 0.0790 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0790 | LR: 4.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8912s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank5 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank5 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0800 | Regression_energy: 0.1039 | Regression_eta: 0.0787 | Regression_pt: 0.1065 | Regression_sin_phi: 0.1316 | LR: 4.94e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.9864s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank5 [2026-10-29 06:18:43,933] INFO : Running validation on rank5 +rank5 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 06:19:32,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:19:33,149] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 06:20:00,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 06:20:01,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 06:20:02,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 06:20:03,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 06:20:13,546] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:20:14,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:20:14,781] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 06:20:30,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 06:20:32,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 06:20:33,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 06:20:34,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 06:20:44,496] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:20:46,294] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 06:21:01,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 06:21:02,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 06:21:03,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 06:21:03,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 06:21:13,782] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 06:25:28,750] INFO : Step 27100/50000 rank5 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2458 | Regression_cos_phi: 0.0766 | Regression_energy: 0.1142 | Regression_eta: 0.0534 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0751 | LR: 4.91e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.7040s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank5 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank5 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2474 | Regression_cos_phi: 0.0932 | Regression_energy: 0.1183 | Regression_eta: 0.0787 | Regression_pt: 0.1221 | Regression_sin_phi: 0.1028 | LR: 4.88e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.5754s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank5 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank5 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0907 | Regression_energy: 0.1072 | Regression_eta: 0.0614 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0777 | LR: 4.85e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.9948s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank5 [2026-10-29 06:37:36,743] INFO : Step 27400/50000 rank5 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2658 | Regression_cos_phi: 0.1425 | Regression_energy: 0.1178 | Regression_eta: 0.0769 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1102 | LR: 4.82e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5114s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank5 [2026-10-29 06:41:28,998] INFO : Step 27500/50000 rank5 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.2739 | Regression_cos_phi: 0.1199 | Regression_energy: 0.1115 | Regression_eta: 0.0790 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1135 | LR: 4.80e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.9226s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank5 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank5 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1102 | Regression_eta: 0.0630 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0940 | LR: 4.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8528s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank5 [2026-10-29 06:49:16,009] INFO : Step 27700/50000 rank5 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.2295 | Regression_cos_phi: 0.1387 | Regression_energy: 0.1174 | Regression_eta: 0.0655 | Regression_pt: 0.1247 | Regression_sin_phi: 0.1021 | LR: 4.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0193s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank5 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank5 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2416 | Regression_cos_phi: 0.0702 | Regression_energy: 0.1027 | Regression_eta: 0.0646 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0789 | LR: 4.71e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.6672s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank5 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank5 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2022 | Regression_cos_phi: 0.1099 | Regression_energy: 0.1156 | Regression_eta: 0.0699 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0734 | LR: 4.68e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5353s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank5 [2026-10-29 07:01:10,307] INFO : Step 28000/50000 rank5 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.2025 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1100 | Regression_eta: 0.0817 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0638 | LR: 4.66e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3471s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank5 [2026-10-29 07:01:11,150] INFO : Running validation on rank5 +rank5 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:02:01,773] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:02:02,348] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 07:02:30,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:02:30,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:02:31,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:02:32,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:02:42,743] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 07:02:43,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:02:44,047] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 07:03:00,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:03:01,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:03:02,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:03:03,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:03:13,939] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:03:14,474] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 07:03:29,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:03:30,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:03:31,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:03:31,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:03:41,978] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 07:08:04,578] INFO : Step 28100/50000 rank5 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2433 | Regression_cos_phi: 0.0801 | Regression_energy: 0.1249 | Regression_eta: 0.0661 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0822 | LR: 4.63e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank5 [2026-10-29 07:11:56,590] INFO : Step 28200/50000 rank5 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2357 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1173 | Regression_eta: 0.0719 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0885 | LR: 4.60e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2152s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank5 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank5 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0991 | Regression_eta: 0.0475 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0732 | LR: 4.57e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.8318s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank5 [2026-10-29 07:20:03,894] INFO : Step 28400/50000 rank5 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2665 | Regression_cos_phi: 0.1156 | Regression_energy: 0.1221 | Regression_eta: 0.0796 | Regression_pt: 0.1289 | Regression_sin_phi: 0.1007 | LR: 4.55e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.0901s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank5 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank5 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2242 | Regression_cos_phi: 0.0991 | Regression_energy: 0.1233 | Regression_eta: 0.0685 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0701 | LR: 4.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1332s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank5 [2026-10-29 07:27:52,535] INFO : Step 28600/50000 rank5 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0609 | Regression_energy: 0.1078 | Regression_eta: 0.0532 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0764 | LR: 4.49e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2894s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank5 [2026-10-29 07:31:49,174] INFO : Step 28700/50000 rank5 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0461 | Regression_energy: 0.1010 | Regression_eta: 0.0573 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0576 | LR: 4.46e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0950s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank5 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank5 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2373 | Regression_cos_phi: 0.1042 | Regression_energy: 0.1197 | Regression_eta: 0.0671 | Regression_pt: 0.1248 | Regression_sin_phi: 0.1182 | LR: 4.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4531s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank5 [2026-10-29 07:39:38,710] INFO : Step 28900/50000 rank5 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1731 | Regression_cos_phi: 0.0913 | Regression_energy: 0.1046 | Regression_eta: 0.0570 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0803 | LR: 4.41e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3199s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank5 [2026-10-29 07:43:43,866] INFO : Step 29000/50000 rank5 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0991 | Regression_eta: 0.0758 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0772 | LR: 4.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9084s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank5 [2026-10-29 07:43:44,820] INFO : Running validation on rank5 +rank5 [2026-10-29 07:43:44,820] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:43:44,820] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:44:33,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:44:33,715] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 07:44:59,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:45:00,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:45:01,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:45:02,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:45:12,397] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 07:45:12,873] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:45:13,372] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 07:45:29,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:45:30,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:45:31,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:45:32,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:45:42,724] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:45:43,718] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 07:45:59,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:45:59,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:46:00,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:46:01,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:46:11,117] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 07:50:42,139] INFO : Step 29100/50000 rank5 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1171 | Regression_energy: 0.1112 | Regression_eta: 0.0748 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0983 | LR: 4.35e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6998s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank5 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank5 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0905 | Regression_eta: 0.0598 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0736 | LR: 4.33e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2669s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank5 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank5 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0938 | Regression_eta: 0.0529 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0797 | LR: 4.30e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0000s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank5 [2026-10-29 08:02:11,407] INFO : Step 29400/50000 rank5 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2492 | Regression_cos_phi: 0.0864 | Regression_energy: 0.1088 | Regression_eta: 0.0783 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0809 | LR: 4.27e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5183s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank5 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank5 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0810 | Regression_energy: 0.1129 | Regression_eta: 0.0814 | Regression_pt: 0.1165 | Regression_sin_phi: 0.1068 | LR: 4.24e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4981s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank5 [2026-10-29 08:09:49,625] INFO : Step 29600/50000 rank5 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0990 | Regression_eta: 0.0583 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0665 | LR: 4.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7086s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank5 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank5 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.2172 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0878 | Regression_eta: 0.0612 | Regression_pt: 0.0909 | Regression_sin_phi: 0.1017 | LR: 4.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1829s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank5 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank5 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.2986 | Regression_cos_phi: 0.1405 | Regression_energy: 0.0977 | Regression_eta: 0.1060 | Regression_pt: 0.1037 | Regression_sin_phi: 0.1438 | LR: 4.16e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3847s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank5 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank5 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0948 | Regression_eta: 0.0776 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0830 | LR: 4.14e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9946s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank5 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank5 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0919 | Regression_energy: 0.1061 | Regression_eta: 0.0902 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0801 | LR: 4.11e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7037s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank5 [2026-10-29 08:24:57,340] INFO : Running validation on rank5 +rank5 [2026-10-29 08:24:57,341] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 08:24:57,341] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 08:25:45,091] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:25:45,651] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 08:26:11,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 08:26:12,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 08:26:13,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 08:26:14,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 08:26:24,144] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 08:26:24,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:26:25,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 08:26:41,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 08:26:42,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 08:26:43,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 08:26:44,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 08:26:54,134] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:26:55,053] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 08:27:10,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 08:27:11,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 08:27:11,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 08:27:12,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 08:27:22,555] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank5 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2265 | Regression_cos_phi: 0.0748 | Regression_energy: 0.1027 | Regression_eta: 0.0572 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0682 | LR: 4.08e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank5 [2026-10-29 08:35:23,905] INFO : Step 30200/50000 rank5 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2115 | Regression_cos_phi: 0.0711 | Regression_energy: 0.1067 | Regression_eta: 0.0408 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0614 | LR: 4.06e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5330s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank5 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank5 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0899 | Regression_eta: 0.0503 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0880 | LR: 4.03e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1812s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank5 [2026-10-29 08:43:19,168] INFO : Step 30400/50000 rank5 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0992 | Regression_eta: 0.0851 | Regression_pt: 0.1033 | Regression_sin_phi: 0.1038 | LR: 4.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5070s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank5 [2026-10-29 08:47:03,581] INFO : Step 30500/50000 rank5 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0921 | Regression_eta: 0.0634 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0922 | LR: 3.98e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7634s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank5 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank5 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0830 | Regression_eta: 0.0523 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0614 | LR: 3.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5468s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank5 [2026-10-29 08:54:35,590] INFO : Step 30700/50000 rank5 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0886 | Regression_eta: 0.0589 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0670 | LR: 3.92e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank5 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank5 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2461 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1080 | Regression_eta: 0.0785 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0919 | LR: 3.90e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2065s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank5 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank5 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2317 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0918 | Regression_eta: 0.0725 | Regression_pt: 0.0946 | Regression_sin_phi: 0.1287 | LR: 3.87e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.9298s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank5 [2026-10-29 09:05:52,365] INFO : Step 31000/50000 rank5 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0828 | Regression_eta: 0.0496 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0818 | LR: 3.84e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1880s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank5 [2026-10-29 09:05:53,358] INFO : Running validation on rank5 +rank5 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:06:41,699] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:06:42,226] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 09:07:08,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:07:09,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:07:10,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:07:11,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:07:21,382] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:07:22,581] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 09:07:38,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:07:39,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:07:40,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:07:41,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:07:51,338] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:07:52,509] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 09:08:07,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:08:08,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:08:09,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:08:09,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:08:19,778] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank5 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2348 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0900 | Regression_eta: 0.0881 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0945 | LR: 3.82e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.4611s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank5 [2026-10-29 09:16:29,207] INFO : Step 31200/50000 rank5 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0768 | Regression_energy: 0.1089 | Regression_eta: 0.0505 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0762 | LR: 3.79e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5351s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank5 [2026-10-29 09:20:24,830] INFO : Step 31300/50000 rank5 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0743 | Regression_eta: 0.0514 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0398 | LR: 3.76e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.4682s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank5 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank5 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0900 | Regression_eta: 0.0694 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0847 | LR: 3.74e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8376s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank5 [2026-10-29 09:28:17,181] INFO : Step 31500/50000 rank5 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2383 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1225 | Regression_eta: 0.0939 | Regression_pt: 0.1277 | Regression_sin_phi: 0.1094 | LR: 3.71e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.3626s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank5 [2026-10-29 09:32:16,797] INFO : Step 31600/50000 rank5 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0929 | Regression_eta: 0.0843 | Regression_pt: 0.0948 | Regression_sin_phi: 0.1012 | LR: 3.69e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8615s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank5 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank5 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0773 | Regression_eta: 0.0672 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0725 | LR: 3.66e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1362s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank5 [2026-10-29 09:40:22,623] INFO : Step 31800/50000 rank5 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2391 | Regression_cos_phi: 0.1074 | Regression_energy: 0.0836 | Regression_eta: 0.1060 | Regression_pt: 0.0891 | Regression_sin_phi: 0.1277 | LR: 3.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3538s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank5 [2026-10-29 09:48:15,353] INFO : Step 31900/50000 rank5 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.2152 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0822 | Regression_eta: 0.0506 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0728 | LR: 3.61e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8686s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank5 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank5 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1611 | Regression_cos_phi: 0.1230 | Regression_energy: 0.0829 | Regression_eta: 0.0579 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0798 | LR: 3.58e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1228s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank5 [2026-10-29 09:52:26,792] INFO : Running validation on rank5 +rank5 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:53:17,425] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 09:53:46,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:53:47,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:53:48,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:53:49,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:53:59,950] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 09:54:00,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:54:01,009] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 09:54:17,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:54:18,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:54:19,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:54:20,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:54:30,163] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:54:30,788] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 09:54:46,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:54:47,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:54:47,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:54:48,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:54:58,574] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank5 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0810 | Regression_eta: 0.0605 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0595 | LR: 3.56e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.9467s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank5 [2026-10-29 10:03:26,459] INFO : Step 32200/50000 rank5 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.2152 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0796 | Regression_eta: 0.0747 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0739 | LR: 3.53e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5227s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank5 [2026-10-29 10:07:22,792] INFO : Step 32300/50000 rank5 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0789 | Regression_eta: 0.0596 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0611 | LR: 3.51e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5877s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank5 [2026-10-29 10:11:42,985] INFO : Step 32400/50000 rank5 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0740 | Regression_eta: 0.0525 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0803 | LR: 3.48e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.0247s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank5 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank5 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0876 | Regression_eta: 0.0490 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0570 | LR: 3.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5249s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank5 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank5 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0766 | Regression_eta: 0.0744 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0896 | LR: 3.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4751s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank5 [2026-10-29 10:24:11,749] INFO : Step 32700/50000 rank5 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0690 | Regression_eta: 0.0607 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0604 | LR: 3.41e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0503s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank5 [2026-10-29 10:28:05,809] INFO : Step 32800/50000 rank5 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2094 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0778 | Regression_eta: 0.0498 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0730 | LR: 3.38e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5525s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank5 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank5 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2252 | Regression_cos_phi: 0.1029 | Regression_energy: 0.0834 | Regression_eta: 0.0768 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1301 | LR: 3.36e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8079s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank5 [2026-10-29 10:36:14,933] INFO : Step 33000/50000 rank5 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1641 | Regression_cos_phi: 0.1053 | Regression_energy: 0.0931 | Regression_eta: 0.0632 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0764 | LR: 3.33e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7655s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank5 [2026-10-29 10:36:15,962] INFO : Running validation on rank5 +rank5 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 10:37:03,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:37:04,371] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 10:37:32,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 10:37:34,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 10:37:35,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 10:37:37,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 10:37:47,486] INFO : Time taken to make predictions on device 5 is: 0.30 min +rank5 [2026-10-29 10:37:48,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:37:48,814] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 10:38:04,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 10:38:06,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 10:38:07,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 10:38:08,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 10:38:18,133] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:38:21,705] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 10:38:37,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 10:38:38,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 10:38:38,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 10:38:39,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 10:38:49,526] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 10:43:12,059] INFO : Step 33100/50000 rank5 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0718 | Regression_eta: 0.0426 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0835 | LR: 3.31e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.9493s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank5 [2026-10-29 10:47:22,843] INFO : Step 33200/50000 rank5 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2458 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0830 | Regression_eta: 0.0598 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0798 | LR: 3.28e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6964s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank5 [2026-10-29 10:51:33,085] INFO : Step 33300/50000 rank5 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0772 | Regression_eta: 0.0427 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0490 | LR: 3.26e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4505s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank5 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank5 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0911 | Regression_eta: 0.0634 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0721 | LR: 3.23e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5447s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank5 [2026-10-29 10:59:48,623] INFO : Step 33500/50000 rank5 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1863 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0886 | Regression_eta: 0.0548 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0689 | LR: 3.21e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.9171s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank5 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank5 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0907 | Regression_eta: 0.0609 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0609 | LR: 3.19e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.2017s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank5 [2026-10-29 11:09:08,678] INFO : Step 33700/50000 rank5 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0799 | Regression_eta: 0.0700 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0545 | LR: 3.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3164s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank5 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank5 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2624 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0921 | Regression_eta: 0.0761 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0753 | LR: 3.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3672s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank5 [2026-10-29 11:16:52,540] INFO : Step 33900/50000 rank5 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0848 | Regression_eta: 0.0792 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0937 | LR: 3.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8122s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank5 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank5 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0823 | Regression_eta: 0.0539 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0718 | LR: 3.09e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9493s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank5 [2026-10-29 11:20:48,852] INFO : Running validation on rank5 +rank5 [2026-10-29 11:20:48,853] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 11:20:48,853] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 11:21:42,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:21:42,911] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 11:22:09,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 11:22:10,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 11:22:10,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 11:22:11,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 11:22:21,862] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 11:22:22,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:22:22,981] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 11:22:39,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 11:22:40,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 11:22:41,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 11:22:42,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 11:22:52,084] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:22:53,479] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 11:23:09,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 11:23:09,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 11:23:10,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 11:23:11,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 11:23:21,363] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 11:27:42,517] INFO : Step 34100/50000 rank5 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0950 | Regression_energy: 0.0761 | Regression_eta: 0.0766 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0703 | LR: 3.06e-04 | DataLoad Time: 0.0164s | Model Forward Time: 7.5180s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank5 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank5 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0737 | Regression_eta: 0.0737 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0800 | LR: 3.04e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7874s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank5 [2026-10-29 11:35:56,836] INFO : Step 34300/50000 rank5 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0708 | Regression_eta: 0.0441 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0707 | LR: 3.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 7.2640s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank5 [2026-10-29 11:40:08,373] INFO : Step 34400/50000 rank5 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0700 | Regression_eta: 0.0503 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0670 | LR: 2.99e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.9083s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank5 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank5 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0808 | Regression_eta: 0.0468 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0693 | LR: 2.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5079s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank5 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank5 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0713 | Regression_eta: 0.0742 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0789 | LR: 2.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6058s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank5 [2026-10-29 11:52:00,074] INFO : Step 34700/50000 rank5 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0729 | Regression_eta: 0.0633 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0847 | LR: 2.92e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5538s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank5 [2026-10-29 11:55:53,802] INFO : Step 34800/50000 rank5 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0762 | Regression_eta: 0.0541 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0724 | LR: 2.90e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8583s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank5 [2026-10-29 11:59:58,708] INFO : Step 34900/50000 rank5 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0834 | Regression_eta: 0.0411 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0558 | LR: 2.88e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1697s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank5 [2026-10-29 12:04:06,317] INFO : Step 35000/50000 rank5 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0749 | Regression_eta: 0.0514 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0761 | LR: 2.85e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4571s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank5 [2026-10-29 12:04:07,348] INFO : Running validation on rank5 +rank5 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 12:05:04,844] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:05:05,388] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 12:05:31,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 12:05:32,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 12:05:33,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 12:05:34,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 12:05:44,649] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 12:05:45,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 12:06:02,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 12:06:03,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 12:06:04,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 12:06:05,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 12:06:15,161] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 12:06:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:06:16,084] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 12:06:31,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 12:06:32,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 12:06:32,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 12:06:33,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 12:06:43,672] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank5 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0838 | Regression_eta: 0.0490 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0628 | LR: 2.83e-04 | DataLoad Time: 0.0462s | Model Forward Time: 5.6273s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank5 [2026-10-29 12:15:06,116] INFO : Step 35200/50000 rank5 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0766 | Regression_eta: 0.0441 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0828 | LR: 2.81e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.0574s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank5 [2026-10-29 12:18:57,446] INFO : Step 35300/50000 rank5 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0939 | Regression_eta: 0.0509 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0750 | LR: 2.79e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8164s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank5 [2026-10-29 12:23:00,202] INFO : Step 35400/50000 rank5 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0411 | Regression_energy: 0.1079 | Regression_eta: 0.0475 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0685 | LR: 2.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3950s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank5 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank5 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2297 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0930 | Regression_eta: 0.0735 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0859 | LR: 2.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2678s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank5 [2026-10-29 12:31:17,345] INFO : Step 35600/50000 rank5 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1787 | Regression_cos_phi: 0.1114 | Regression_energy: 0.0889 | Regression_eta: 0.0439 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0726 | LR: 2.72e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.1597s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank5 [2026-10-29 12:35:31,257] INFO : Step 35700/50000 rank5 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0723 | Regression_eta: 0.0413 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0584 | LR: 2.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2775s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank5 [2026-10-29 14:49:26,336] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-29 14:49:26,340] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank5 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-29 14:49:53,254] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-29 14:49:53,273] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-29 14:49:53,273] INFO : nn0 parameters: 171776 +rank5 [2026-10-29 14:49:53,273] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank5 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 1 +rank5 [2026-10-29 14:49:53,279] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,284] INFO : Initializing backbone layer 2 +rank5 [2026-10-29 14:49:53,284] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,290] INFO : Initializing backbone layer 3 +rank5 [2026-10-29 14:49:53,290] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,295] INFO : Initializing backbone layer 4 +rank5 [2026-10-29 14:49:53,295] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,300] INFO : Initializing backbone layer 5 +rank5 [2026-10-29 14:49:53,300] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:49:53,305] INFO : Backbone initialization took 0.03s +rank5 [2026-10-29 14:49:53,306] INFO : backbone parameters: 2374656 +rank5 [2026-10-29 14:49:53,306] INFO : Initializing output DNNs +rank5 [2026-10-29 14:49:53,359] INFO : Output DNNs initialization took 0.05s +rank5 [2026-10-29 14:49:53,359] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-29 14:49:53,359] INFO : jagged_attention enabled=True +rank5 [2026-10-29 14:49:53,359] INFO : flash_attn_varlen enabled=True +rank5 [2026-10-29 14:49:53,359] INFO : task_queries enabled=False +rank5 [2026-10-29 14:49:53,359] INFO : classification_norm parameters: 0 +rank5 [2026-10-29 14:49:53,359] INFO : regression_norm parameters: 0 +rank5 [2026-10-29 14:49:53,359] INFO : classification_query parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : regression_query parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : classification_readout parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : regression_readout parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : output_mode=set +rank5 [2026-10-29 14:49:53,360] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-29 14:49:53,360] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_pid parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_pt parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_eta parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-29 14:49:53,360] INFO : nn_energy parameters: 0 +rank5 [2026-10-29 14:49:53,361] INFO : Total MLPF parameters: 7260244 +rank5 [2026-10-29 14:49:53,361] INFO : MLPF __init__ done +rank5 [2026-10-29 14:49:53,361] INFO : Instantiated model +rank5 [2026-10-29 14:49:53,361] INFO : Moving model to device rank=5 +rank5 [2026-10-29 14:49:53,376] INFO : Moved model to device rank=5 +rank5 [2026-10-29 14:49:53,378] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-29 14:49:53,694] INFO : Restoring training from step 35001 +rank5 [2026-10-29 14:49:53,700] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank5 [2026-10-29 14:49:53,700] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-29 14:49:53,912] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-29 14:49:54,028] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank5 [2026-10-29 14:49:54,028] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-29 14:49:54,029] INFO : Trainable parameters: 7260244 +rank5 [2026-10-29 14:49:54,029] INFO : Non-trainable parameters: 0 +rank5 [2026-10-29 14:49:54,029] INFO : Total parameters: 7260244 +rank5 [2026-10-29 14:49:54,032] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-29 14:49:54,032] INFO : Getting dataloaders +rank5 [2026-10-29 14:49:54,032] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:01,383] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:02,342] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:03,544] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-29 14:50:03,544] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-29 14:50:03,544] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:03,830] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:04,100] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:04,390] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-29 14:50:04,390] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-29 14:50:18,391] INFO : Got dataloaders +rank5 [2026-10-29 14:50:18,392] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank5 [2026-10-29 14:50:18,392] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank5 [2026-10-29 14:50:18,392] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank5 [2026-10-29 14:50:18,392] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank5 [2026-10-29 14:50:19,745] INFO : loader split=train rank=5 len=21108 +rank5 [2026-10-29 14:50:19,745] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-29 14:50:19,745] INFO : Starting training from step 35001 to 50000 on rank 5 of 8 +rank5 [2026-10-29 14:50:19,745] INFO : Creating train_iterator +rank5 [2026-10-29 14:50:19,745] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank5 [2026-10-29 14:50:33,778] INFO : Created train_iterator +rank5 [2026-10-29 14:54:16,363] INFO : Step 35100/50000 rank5 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1696 | Regression_cos_phi: 0.1042 | Regression_energy: 0.0842 | Regression_eta: 0.0528 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0691 | LR: 2.83e-04 | DataLoad Time: 0.0161s | Model Forward Time: 2.8682s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank5 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank5 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.2116 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0799 | Regression_eta: 0.0534 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0918 | LR: 2.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.5100s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank5 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank5 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0867 | Regression_eta: 0.0585 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0737 | LR: 2.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5918s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank5 [2026-10-29 15:05:10,669] INFO : Step 35400/50000 rank5 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0427 | Regression_energy: 0.1110 | Regression_eta: 0.0498 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0728 | LR: 2.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4263s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank5 [2026-10-29 15:08:49,316] INFO : Step 35500/50000 rank5 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0926 | Regression_eta: 0.0585 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0813 | LR: 2.74e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.4540s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank5 [2026-10-29 15:12:27,419] INFO : Step 35600/50000 rank5 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0845 | Regression_eta: 0.0463 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0736 | LR: 2.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6874s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank5 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank5 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0747 | Regression_eta: 0.0498 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0818 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.3517s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank5 [2026-10-29 15:19:47,616] INFO : Step 35800/50000 rank5 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0774 | Regression_eta: 0.0498 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0510 | LR: 2.68e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.4879s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank5 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank5 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0888 | Regression_eta: 0.0480 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0711 | LR: 2.65e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8626s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank5 [2026-10-29 15:27:19,181] INFO : Step 36000/50000 rank5 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0888 | Regression_eta: 0.0543 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0752 | LR: 2.63e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8279s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank5 [2026-10-29 15:27:20,114] INFO : Running validation on rank5 +rank5 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 15:28:08,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:28:08,877] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 15:28:36,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 15:28:37,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 15:28:38,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 15:28:39,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 15:28:49,252] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 15:28:49,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:28:50,425] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 15:29:06,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 15:29:07,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 15:29:08,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 15:29:09,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 15:29:19,333] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:29:20,197] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 15:29:35,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 15:29:36,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 15:29:36,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 15:29:37,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 15:29:47,600] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank5 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0956 | Regression_eta: 0.0560 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0508 | LR: 2.61e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.9085s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank5 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank5 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2303 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0854 | Regression_eta: 0.0751 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0858 | LR: 2.59e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5677s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank5 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank5 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0918 | Regression_eta: 0.0514 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0611 | LR: 2.57e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5111s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank5 [2026-10-29 15:45:12,807] INFO : Step 36400/50000 rank5 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0763 | Regression_eta: 0.0515 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0649 | LR: 2.55e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.2440s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank5 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank5 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0793 | Regression_eta: 0.0572 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0601 | LR: 2.52e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0157s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank5 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank5 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0983 | Regression_eta: 0.0593 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0741 | LR: 2.50e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7030s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank5 [2026-10-29 15:56:51,301] INFO : Step 36700/50000 rank5 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0838 | Regression_eta: 0.0698 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0749 | LR: 2.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7862s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank5 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank5 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0841 | Regression_eta: 0.0427 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0659 | LR: 2.46e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.4177s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank5 [2026-10-29 16:04:39,810] INFO : Step 36900/50000 rank5 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0841 | Regression_eta: 0.0599 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0854 | LR: 2.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank5 [2026-10-29 16:08:30,145] INFO : Step 37000/50000 rank5 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0890 | Regression_eta: 0.0712 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0677 | LR: 2.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9053s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank5 [2026-10-29 16:08:31,022] INFO : Running validation on rank5 +rank5 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:09:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:09:20,760] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 16:09:46,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:09:47,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:09:48,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:09:49,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:09:59,152] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 16:10:15,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:10:16,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:10:17,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:10:19,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:10:29,026] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:10:29,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:10:30,425] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 16:10:45,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:10:46,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:10:47,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:10:47,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:10:57,899] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 16:15:06,403] INFO : Step 37100/50000 rank5 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0686 | Regression_eta: 0.0467 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0771 | LR: 2.40e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8498s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank5 [2026-10-29 16:18:46,283] INFO : Step 37200/50000 rank5 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0825 | Regression_eta: 0.0648 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0942 | LR: 2.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2308s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank5 [2026-10-29 16:22:16,730] INFO : Step 37300/50000 rank5 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0887 | Regression_eta: 0.0564 | Regression_pt: 0.0928 | Regression_sin_phi: 0.1030 | LR: 2.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5819s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank5 [2026-10-29 16:26:01,824] INFO : Step 37400/50000 rank5 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0783 | Regression_eta: 0.0447 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0899 | LR: 2.34e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.9440s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank5 [2026-10-29 16:30:27,636] INFO : Step 37500/50000 rank5 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0836 | Regression_eta: 0.0477 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0523 | LR: 2.32e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3481s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank5 [2026-10-29 16:34:14,939] INFO : Step 37600/50000 rank5 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0840 | Regression_energy: 0.0958 | Regression_eta: 0.0570 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0669 | LR: 2.30e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1114s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank5 [2026-10-29 16:38:00,807] INFO : Step 37700/50000 rank5 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1951 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0770 | Regression_eta: 0.0739 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0958 | LR: 2.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1655s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank5 [2026-10-29 16:41:51,410] INFO : Step 37800/50000 rank5 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0844 | Regression_eta: 0.0746 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0994 | LR: 2.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.2926s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank5 [2026-10-29 16:45:39,813] INFO : Step 37900/50000 rank5 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2596 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0886 | Regression_eta: 0.0662 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0852 | LR: 2.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8088s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank5 [2026-10-29 16:49:32,192] INFO : Step 38000/50000 rank5 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0871 | Regression_eta: 0.0849 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0854 | LR: 2.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8888s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank5 [2026-10-29 16:49:33,136] INFO : Running validation on rank5 +rank5 [2026-10-29 16:49:33,137] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:49:33,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:50:21,275] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 16:50:48,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:50:49,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:50:51,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:50:53,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:51:03,429] INFO : Time taken to make predictions on device 5 is: 0.29 min +rank5 [2026-10-29 16:51:06,261] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:51:06,744] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 16:51:22,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:51:23,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:51:24,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:51:25,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:51:35,603] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:51:36,911] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 16:51:52,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:51:53,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:51:53,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:51:54,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:52:04,410] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank5 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0888 | Regression_eta: 0.0530 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0460 | LR: 2.20e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3701s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank5 [2026-10-29 17:00:01,012] INFO : Step 38200/50000 rank5 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2005 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0818 | Regression_eta: 0.0625 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0717 | LR: 2.18e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8300s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank5 [2026-10-29 17:03:44,996] INFO : Step 38300/50000 rank5 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2400 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0770 | Regression_eta: 0.0780 | Regression_pt: 0.0788 | Regression_sin_phi: 0.1120 | LR: 2.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2337s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank5 [2026-10-29 17:07:28,967] INFO : Step 38400/50000 rank5 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0716 | Regression_eta: 0.0646 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0522 | LR: 2.14e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.5969s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank5 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank5 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0866 | Regression_eta: 0.0474 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0682 | LR: 2.12e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.5369s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank5 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank5 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0770 | Regression_eta: 0.0397 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0725 | LR: 2.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4122s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank5 [2026-10-29 17:18:55,038] INFO : Step 38700/50000 rank5 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0738 | Regression_eta: 0.0419 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0712 | LR: 2.09e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.8750s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank5 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank5 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0949 | Regression_eta: 0.0410 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0548 | LR: 2.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5788s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank5 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank5 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0801 | Regression_eta: 0.0673 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0701 | LR: 2.05e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6642s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank5 [2026-10-29 17:30:46,440] INFO : Step 39000/50000 rank5 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0627 | Regression_eta: 0.0647 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0649 | LR: 2.03e-04 | DataLoad Time: 0.0227s | Model Forward Time: 3.6460s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank5 [2026-10-29 17:30:47,444] INFO : Running validation on rank5 +rank5 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 17:31:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:31:36,208] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 17:32:01,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 17:32:02,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 17:32:03,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 17:32:04,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 17:32:14,632] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:32:15,958] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 17:32:32,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 17:32:33,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 17:32:34,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 17:32:34,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 17:32:44,981] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:32:45,863] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 17:33:01,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 17:33:01,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 17:33:02,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 17:33:03,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 17:33:13,382] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank5 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0737 | Regression_eta: 0.0748 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0931 | LR: 2.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6371s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank5 [2026-10-29 17:41:18,055] INFO : Step 39200/50000 rank5 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0720 | Regression_eta: 0.0417 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0499 | LR: 2.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7441s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank5 [2026-10-29 17:45:05,747] INFO : Step 39300/50000 rank5 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0704 | Regression_eta: 0.0753 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0839 | LR: 1.98e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.9364s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank5 [2026-10-29 17:49:01,213] INFO : Step 39400/50000 rank5 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0700 | Regression_eta: 0.0691 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0628 | LR: 1.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0460s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank5 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank5 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0801 | Regression_eta: 0.0432 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0782 | LR: 1.94e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2576s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank5 [2026-10-29 17:56:59,043] INFO : Step 39600/50000 rank5 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2159 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0762 | Regression_eta: 0.0501 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0611 | LR: 1.93e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1942s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank5 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank5 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0739 | Regression_eta: 0.0542 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0718 | LR: 1.91e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2342s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank5 [2026-10-29 18:04:32,041] INFO : Step 39800/50000 rank5 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0717 | Regression_eta: 0.0416 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0561 | LR: 1.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9000s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank5 [2026-10-29 18:08:19,424] INFO : Step 39900/50000 rank5 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0721 | Regression_eta: 0.0538 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0803 | LR: 1.88e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank5 [2026-10-29 18:12:05,434] INFO : Step 40000/50000 rank5 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0790 | Regression_eta: 0.0608 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0692 | LR: 1.86e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9413s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank5 [2026-10-29 18:12:06,328] INFO : Running validation on rank5 +rank5 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:12:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:12:59,651] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 18:13:23,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:13:24,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:13:25,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:13:26,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:13:36,284] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 18:13:37,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:13:37,721] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 18:13:53,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:13:54,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:13:55,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:13:56,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:14:06,576] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:14:07,478] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 18:14:22,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:14:23,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:14:24,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:14:25,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:14:35,114] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 18:18:48,523] INFO : Step 40100/50000 rank5 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1903 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0884 | Regression_eta: 0.0537 | Regression_pt: 0.0924 | Regression_sin_phi: 0.1007 | LR: 1.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1848s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank5 [2026-10-29 18:22:51,410] INFO : Step 40200/50000 rank5 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0855 | Regression_eta: 0.0649 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0801 | LR: 1.83e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.9279s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank5 [2026-10-29 18:26:53,014] INFO : Step 40300/50000 rank5 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2394 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1020 | Regression_eta: 0.0690 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0923 | LR: 1.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4858s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank5 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank5 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0751 | Regression_eta: 0.0662 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0788 | LR: 1.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0119s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank5 [2026-10-29 18:34:31,871] INFO : Step 40500/50000 rank5 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0975 | Regression_eta: 0.0452 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0620 | LR: 1.78e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.4810s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank5 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank5 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0878 | Regression_eta: 0.0526 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0627 | LR: 1.76e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1580s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank5 [2026-10-29 18:42:09,670] INFO : Step 40700/50000 rank5 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0854 | Regression_eta: 0.0389 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0696 | LR: 1.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5728s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank5 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank5 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0657 | Regression_energy: 0.1017 | Regression_eta: 0.0536 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0740 | LR: 1.73e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.0697s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank5 [2026-10-29 18:49:49,167] INFO : Step 40900/50000 rank5 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0759 | Regression_eta: 0.0405 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0866 | LR: 1.72e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.6414s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank5 [2026-10-29 18:53:41,415] INFO : Step 41000/50000 rank5 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0798 | Regression_eta: 0.0629 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0619 | LR: 1.70e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4686s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank5 [2026-10-29 18:53:42,384] INFO : Running validation on rank5 +rank5 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:54:33,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:54:33,854] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 18:54:59,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:55:00,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:55:01,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:55:02,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:55:12,460] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:55:13,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:55:13,607] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 18:55:29,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:55:30,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:55:31,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:55:32,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:55:42,891] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:55:43,852] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 18:55:59,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:56:00,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:56:00,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:56:01,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:56:11,718] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 19:00:33,893] INFO : Step 41100/50000 rank5 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0278 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0789 | Regression_eta: 0.0365 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0584 | LR: 1.69e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8497s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank5 [2026-10-29 19:04:28,229] INFO : Step 41200/50000 rank5 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2170 | Regression_cos_phi: 0.0781 | Regression_energy: 0.1007 | Regression_eta: 0.0549 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0702 | LR: 1.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9459s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank5 [2026-10-29 19:08:18,073] INFO : Step 41300/50000 rank5 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0904 | Regression_eta: 0.0546 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0938 | LR: 1.66e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1414s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank5 [2026-10-29 19:12:02,347] INFO : Step 41400/50000 rank5 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0868 | Regression_eta: 0.0776 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0955 | LR: 1.64e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9384s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank5 [2026-10-29 19:15:48,707] INFO : Step 41500/50000 rank5 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0711 | Regression_eta: 0.0486 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0607 | LR: 1.63e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.3264s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank5 [2026-10-29 19:19:48,124] INFO : Step 41600/50000 rank5 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2095 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0839 | Regression_eta: 0.0653 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0757 | LR: 1.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7652s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank5 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank5 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0840 | Regression_eta: 0.0472 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0616 | LR: 1.60e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9477s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank5 [2026-10-29 19:27:38,848] INFO : Step 41800/50000 rank5 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0838 | Regression_eta: 0.0535 | Regression_pt: 0.0867 | Regression_sin_phi: 0.1024 | LR: 1.58e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.3185s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank5 [2026-10-29 19:31:32,457] INFO : Step 41900/50000 rank5 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1881 | Regression_cos_phi: 0.1155 | Regression_energy: 0.0807 | Regression_eta: 0.0491 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0826 | LR: 1.57e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.4524s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank5 [2026-10-29 19:35:32,890] INFO : Step 42000/50000 rank5 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2251 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0854 | Regression_eta: 0.0593 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0748 | LR: 1.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9395s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank5 [2026-10-29 19:35:33,871] INFO : Running validation on rank5 +rank5 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 19:36:22,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:36:23,273] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 19:36:50,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 19:36:51,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 19:36:51,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 19:36:52,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 19:37:02,892] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:37:04,354] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 19:37:19,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 19:37:20,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 19:37:21,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 19:37:22,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 19:37:32,823] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:37:33,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:37:34,030] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 19:37:49,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 19:37:49,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 19:37:50,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 19:37:51,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 19:38:01,136] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 19:42:30,056] INFO : Step 42100/50000 rank5 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0892 | Regression_eta: 0.0407 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0707 | LR: 1.54e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0595s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank5 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank5 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0892 | Regression_eta: 0.0465 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0660 | LR: 1.53e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8375s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank5 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank5 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 19:50:51,513] INFO : Step 42300/50000 rank5 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2054 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0738 | Regression_eta: 0.0454 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0712 | LR: 1.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2021s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank5 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank5 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0776 | Regression_energy: 0.1061 | Regression_eta: 0.0487 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0979 | LR: 1.50e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.6743s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank5 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank5 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2200 | Regression_cos_phi: 0.1010 | Regression_energy: 0.1050 | Regression_eta: 0.0715 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1237 | LR: 1.49e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.1952s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank5 [2026-10-29 20:02:59,632] INFO : Step 42600/50000 rank5 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2379 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0919 | Regression_eta: 0.0760 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0680 | LR: 1.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 8.8081s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank5 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank5 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0829 | Regression_energy: 0.1024 | Regression_eta: 0.0685 | Regression_pt: 0.1059 | Regression_sin_phi: 0.0912 | LR: 1.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7995s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank5 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank5 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0927 | Regression_eta: 0.0459 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0670 | LR: 1.45e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5859s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank5 [2026-10-29 20:14:51,335] INFO : Step 42900/50000 rank5 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0943 | Regression_eta: 0.0539 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0716 | LR: 1.44e-04 | DataLoad Time: 0.0113s | Model Forward Time: 4.4227s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank5 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank5 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.2079 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0880 | Regression_eta: 0.0600 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0775 | LR: 1.43e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9444s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank5 [2026-10-29 20:18:48,627] INFO : Running validation on rank5 +rank5 [2026-10-29 20:18:48,628] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 20:18:48,628] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 20:19:38,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:19:39,385] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 20:20:09,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 20:20:10,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 20:20:11,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 20:20:12,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 20:20:22,521] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:20:23,703] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 20:20:40,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 20:20:41,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 20:20:42,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 20:20:43,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 20:20:53,020] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:20:53,998] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 20:21:09,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 20:21:09,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 20:21:10,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 20:21:11,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 20:21:21,161] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank5 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0753 | Regression_eta: 0.0484 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0871 | LR: 1.42e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.2851s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank5 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank5 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.2121 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0864 | Regression_eta: 0.0592 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0958 | LR: 1.40e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1811s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank5 [2026-10-29 20:33:23,251] INFO : Step 43300/50000 rank5 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2206 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0967 | Regression_eta: 0.0493 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0626 | LR: 1.39e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3879s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank5 [2026-10-29 20:37:09,705] INFO : Step 43400/50000 rank5 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0804 | Regression_eta: 0.0457 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0729 | LR: 1.38e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5490s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank5 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank5 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0746 | Regression_eta: 0.0509 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0622 | LR: 1.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1835s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank5 [2026-10-29 20:44:45,321] INFO : Step 43600/50000 rank5 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0958 | Regression_eta: 0.0527 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0721 | LR: 1.36e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0822s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank5 [2026-10-29 20:48:25,993] INFO : Step 43700/50000 rank5 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0852 | Regression_eta: 0.0446 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0803 | LR: 1.35e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.4612s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank5 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank5 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0848 | Regression_eta: 0.0667 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0580 | LR: 1.34e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.5352s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank5 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank5 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2247 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0805 | Regression_eta: 0.0518 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0853 | LR: 1.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3474s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank5 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank5 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2225 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0815 | Regression_eta: 0.0647 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0709 | LR: 1.32e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1958s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank5 [2026-10-29 21:00:02,713] INFO : Running validation on rank5 +rank5 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:00:02,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:00:57,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:00:58,022] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 21:01:24,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:01:25,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:01:26,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:01:27,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:01:37,445] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:01:38,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:01:38,954] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 21:01:54,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:01:55,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:01:56,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:01:57,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:02:07,734] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:02:08,956] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 21:02:24,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:02:25,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:02:25,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:02:26,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:02:36,604] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 21:06:42,683] INFO : Step 44100/50000 rank5 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0837 | Regression_eta: 0.0486 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0922 | LR: 1.31e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank5 [2026-10-29 21:10:40,736] INFO : Step 44200/50000 rank5 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0694 | Regression_eta: 0.0454 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0669 | LR: 1.30e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9771s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank5 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank5 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0801 | Regression_eta: 0.0625 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0550 | LR: 1.29e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1191s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank5 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank5 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0736 | Regression_eta: 0.0589 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0714 | LR: 1.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7689s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank5 [2026-10-29 21:22:16,364] INFO : Step 44500/50000 rank5 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0879 | Regression_eta: 0.0493 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0931 | LR: 1.27e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.8112s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank5 [2026-10-29 21:26:10,594] INFO : Step 44600/50000 rank5 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0855 | Regression_eta: 0.0434 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0637 | LR: 1.26e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1490s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank5 [2026-10-29 21:30:16,075] INFO : Step 44700/50000 rank5 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0756 | Regression_eta: 0.0485 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0773 | LR: 1.25e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6984s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank5 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank5 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0861 | Regression_eta: 0.0461 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0749 | LR: 1.24e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4087s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank5 [2026-10-29 21:38:11,005] INFO : Step 44900/50000 rank5 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0856 | Regression_eta: 0.0385 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0572 | LR: 1.23e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9524s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank5 [2026-10-29 21:42:13,646] INFO : Step 45000/50000 rank5 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0725 | Regression_eta: 0.0730 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0643 | LR: 1.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0114s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank5 [2026-10-29 21:42:14,559] INFO : Running validation on rank5 +rank5 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:43:06,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:43:07,018] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 21:43:32,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:43:33,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:43:34,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:43:35,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:43:45,648] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:43:47,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:43:47,769] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 21:44:03,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:44:04,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:44:05,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:44:06,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:44:16,701] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:44:17,826] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 21:44:33,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:44:34,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:44:34,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:44:35,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:44:45,600] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 21:48:55,616] INFO : Step 45100/50000 rank5 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0407 | Regression_energy: 0.1040 | Regression_eta: 0.0472 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0647 | LR: 1.21e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4104s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank5 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank5 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0880 | Regression_eta: 0.0545 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0736 | LR: 1.20e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1532s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank5 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank5 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2284 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0991 | Regression_eta: 0.0518 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0615 | LR: 1.19e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.0122s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank5 [2026-10-29 22:00:32,928] INFO : Step 45400/50000 rank5 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0909 | Regression_eta: 0.0382 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0717 | LR: 1.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9030s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank5 [2026-10-29 22:04:23,232] INFO : Step 45500/50000 rank5 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0775 | Regression_eta: 0.0421 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0470 | LR: 1.18e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6352s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank5 [2026-10-29 22:08:21,194] INFO : Step 45600/50000 rank5 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0864 | Regression_eta: 0.0519 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0814 | LR: 1.17e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0167s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank5 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank5 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0806 | Regression_eta: 0.0542 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0436 | LR: 1.16e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.0834s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank5 [2026-10-29 22:16:08,602] INFO : Step 45800/50000 rank5 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0803 | Regression_eta: 0.0572 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0810 | LR: 1.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6668s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank5 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank5 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0762 | Regression_eta: 0.0560 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0632 | LR: 1.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9609s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank5 [2026-10-29 22:24:24,450] INFO : Step 46000/50000 rank5 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0825 | Regression_eta: 0.0494 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0602 | LR: 1.14e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.2992s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank5 [2026-10-29 22:24:25,404] INFO : Running validation on rank5 +rank5 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 22:25:16,575] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:25:17,233] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 22:25:46,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 22:25:47,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 22:25:48,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 22:25:49,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 22:25:59,142] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 22:25:59,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:26:00,492] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 22:26:16,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 22:26:17,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 22:26:19,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 22:26:20,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 22:26:30,568] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:26:31,922] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 22:26:47,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 22:26:48,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 22:26:49,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 22:26:49,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 22:26:59,864] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 22:31:16,604] INFO : Step 46100/50000 rank5 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0928 | Regression_eta: 0.0481 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0468 | LR: 1.13e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0440s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank5 [2026-10-29 22:35:12,810] INFO : Step 46200/50000 rank5 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0825 | Regression_eta: 0.0526 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0590 | LR: 1.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9315s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank5 [2026-10-29 22:39:25,151] INFO : Step 46300/50000 rank5 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2257 | Regression_cos_phi: 0.0804 | Regression_energy: 0.1023 | Regression_eta: 0.0791 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0875 | LR: 1.12e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.9860s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank5 [2026-10-29 22:43:23,784] INFO : Step 46400/50000 rank5 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0700 | Regression_eta: 0.0583 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0635 | LR: 1.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6665s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank5 [2026-10-29 22:47:17,350] INFO : Step 46500/50000 rank5 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0721 | Regression_eta: 0.0576 | Regression_pt: 0.0759 | Regression_sin_phi: 0.1040 | LR: 1.11e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5729s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank5 [2026-10-29 22:51:20,583] INFO : Step 46600/50000 rank5 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0836 | Regression_eta: 0.0428 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0598 | LR: 1.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3959s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank5 [2026-10-29 22:55:26,214] INFO : Step 46700/50000 rank5 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0885 | Regression_eta: 0.0594 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0783 | LR: 1.10e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0473s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank5 [2026-10-29 22:59:26,803] INFO : Step 46800/50000 rank5 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0815 | Regression_eta: 0.0540 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0529 | LR: 1.09e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.9155s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank5 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank5 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0818 | Regression_eta: 0.0456 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0645 | LR: 1.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.0408s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank5 [2026-10-29 23:08:15,439] INFO : Step 47000/50000 rank5 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0792 | Regression_eta: 0.0562 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0697 | LR: 1.08e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2726s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank5 [2026-10-29 23:08:16,507] INFO : Running validation on rank5 +rank5 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 23:09:07,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:09:08,185] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 23:09:36,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 23:09:38,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 23:09:40,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 23:09:42,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 23:09:52,272] INFO : Time taken to make predictions on device 5 is: 0.31 min +rank5 [2026-10-29 23:09:55,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:09:55,736] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 23:10:11,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 23:10:12,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 23:10:13,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 23:10:14,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 23:10:24,852] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:10:25,936] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 23:10:41,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 23:10:42,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 23:10:42,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 23:10:43,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 23:10:53,672] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank5 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0814 | Regression_eta: 0.0639 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0976 | LR: 1.07e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7048s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank5 [2026-10-29 23:19:19,774] INFO : Step 47200/50000 rank5 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0718 | Regression_eta: 0.0385 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0516 | LR: 1.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.6705s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank5 [2026-10-29 23:23:31,492] INFO : Step 47300/50000 rank5 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2169 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0981 | Regression_eta: 0.0906 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0993 | LR: 1.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7150s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank5 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank5 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2020 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0904 | Regression_eta: 0.0425 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0575 | LR: 1.06e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0189s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank5 [2026-10-29 23:31:38,583] INFO : Step 47500/50000 rank5 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0800 | Regression_eta: 0.0393 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0543 | LR: 1.06e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8369s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank5 [2026-10-29 23:35:34,355] INFO : Step 47600/50000 rank5 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0747 | Regression_eta: 0.0612 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0873 | LR: 1.05e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.5023s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank5 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank5 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0782 | Regression_eta: 0.0621 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0881 | LR: 1.05e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8403s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank5 [2026-10-29 23:43:53,969] INFO : Step 47800/50000 rank5 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0780 | Regression_eta: 0.0508 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0692 | LR: 1.04e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.6059s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank5 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank5 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0631 | Regression_eta: 0.0591 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0786 | LR: 1.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1956s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank5 [2026-10-29 23:51:38,727] INFO : Step 48000/50000 rank5 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2330 | Regression_cos_phi: 0.1036 | Regression_energy: 0.0763 | Regression_eta: 0.0694 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0714 | LR: 1.04e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2955s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank5 [2026-10-29 23:51:39,733] INFO : Running validation on rank5 +rank5 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 23:52:31,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:52:31,693] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 23:52:58,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 23:53:00,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 23:53:01,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 23:53:02,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 23:53:12,353] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 23:53:13,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 23:53:29,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 23:53:30,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 23:53:31,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 23:53:32,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 23:53:42,490] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:53:43,872] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 23:53:59,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 23:53:59,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 23:54:00,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 23:54:01,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 23:54:11,238] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 00:00:53,533] INFO : Step 48100/50000 rank5 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1822 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0823 | Regression_eta: 0.0559 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0662 | LR: 1.03e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5285s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank5 [2026-10-30 00:04:40,583] INFO : Step 48200/50000 rank5 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0825 | Regression_eta: 0.0340 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0423 | LR: 1.03e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0273s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank5 [2026-10-30 00:08:31,628] INFO : Step 48300/50000 rank5 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0884 | Regression_eta: 0.0452 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0849 | LR: 1.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9449s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank5 [2026-10-30 00:12:19,574] INFO : Step 48400/50000 rank5 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0613 | Regression_eta: 0.0555 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0623 | LR: 1.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3345s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank5 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank5 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0805 | Regression_eta: 0.0638 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0729 | LR: 1.02e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.5859s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank5 [2026-10-30 00:19:50,480] INFO : Step 48600/50000 rank5 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0753 | Regression_eta: 0.0590 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0636 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9681s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank5 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank5 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0792 | Regression_eta: 0.0766 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0881 | LR: 1.02e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.0945s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank5 [2026-10-30 00:27:30,340] INFO : Step 48800/50000 rank5 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0845 | Regression_eta: 0.0574 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0500 | LR: 1.01e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4222s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank5 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank5 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1893 | Regression_cos_phi: 0.1285 | Regression_energy: 0.0772 | Regression_eta: 0.0815 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0854 | LR: 1.01e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6430s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank5 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank5 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0726 | Regression_eta: 0.0490 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0646 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5982s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank5 [2026-10-30 00:35:15,047] INFO : Running validation on rank5 +rank5 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:35:15,048] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:36:03,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 00:36:29,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:36:31,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:36:32,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:36:33,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:36:43,192] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:36:48,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:36:48,746] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 00:37:04,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:37:05,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:37:06,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:37:07,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:37:17,388] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:37:17,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:37:18,486] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 00:37:33,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 00:37:34,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 00:37:35,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 00:37:35,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 00:37:45,760] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 00:41:49,930] INFO : Step 49100/50000 rank5 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0817 | Regression_eta: 0.0565 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0689 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.6181s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank5 [2026-10-30 00:45:59,115] INFO : Step 49200/50000 rank5 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0837 | Regression_eta: 0.0457 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0708 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5906s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank5 [2026-10-30 00:49:54,420] INFO : Step 49300/50000 rank5 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0794 | Regression_eta: 0.0576 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0784 | LR: 1.00e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9854s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank5 [2026-10-30 00:53:44,253] INFO : Step 49400/50000 rank5 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0675 | Regression_eta: 0.0404 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0493 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9627s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank5 [2026-10-30 00:57:39,408] INFO : Step 49500/50000 rank5 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0897 | Regression_eta: 0.0459 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0661 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0108s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank5 [2026-10-30 01:01:39,140] INFO : Step 49600/50000 rank5 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0842 | Regression_eta: 0.0416 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0540 | LR: 1.00e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8021s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank5 [2026-10-30 01:05:30,841] INFO : Step 49700/50000 rank5 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0742 | Regression_eta: 0.0433 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0606 | LR: 1.00e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4577s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank5 [2026-10-30 01:09:21,932] INFO : Step 49800/50000 rank5 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0721 | Regression_eta: 0.0760 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0759 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3601s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank5 [2026-10-30 01:13:13,493] INFO : Step 49900/50000 rank5 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2544 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0797 | Regression_eta: 0.0730 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0926 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0834s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank5 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank5 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0745 | Regression_eta: 0.0523 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0590 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8274s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank5 [2026-10-30 01:17:09,619] INFO : Running validation on rank5 +rank5 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 01:18:02,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:18:03,251] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 01:18:29,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 01:18:30,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 01:18:31,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 01:18:31,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 01:18:41,968] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 01:18:42,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:18:43,092] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 01:18:59,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 01:19:00,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 01:19:01,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 01:19:02,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 01:19:12,269] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 01:19:28,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:19:29,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:19:29,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:19:30,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:19:40,516] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 01:19:41,572] INFO : Training completed. Total time on device 5: 629.364min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.6 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..4295035f392b90cdaf1271f0f98a31b0a95f7072 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.6 @@ -0,0 +1,2483 @@ +rank6 [2026-10-28 11:12:32,475] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-28 11:12:32,476] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank6 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-28 11:12:57,453] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-28 11:12:57,453] INFO : nn0 parameters: 171776 +rank6 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank6 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,459] INFO : Initializing backbone layer 1 +rank6 [2026-10-28 11:12:57,459] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,464] INFO : Initializing backbone layer 2 +rank6 [2026-10-28 11:12:57,464] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,470] INFO : Initializing backbone layer 3 +rank6 [2026-10-28 11:12:57,470] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,475] INFO : Initializing backbone layer 4 +rank6 [2026-10-28 11:12:57,475] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,480] INFO : Initializing backbone layer 5 +rank6 [2026-10-28 11:12:57,480] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:57,485] INFO : Backbone initialization took 0.03s +rank6 [2026-10-28 11:12:57,486] INFO : backbone parameters: 2374656 +rank6 [2026-10-28 11:12:57,486] INFO : Initializing output DNNs +rank6 [2026-10-28 11:12:57,523] INFO : Output DNNs initialization took 0.04s +rank6 [2026-10-28 11:12:57,523] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-28 11:12:57,523] INFO : jagged_attention enabled=True +rank6 [2026-10-28 11:12:57,523] INFO : flash_attn_varlen enabled=True +rank6 [2026-10-28 11:12:57,523] INFO : task_queries enabled=False +rank6 [2026-10-28 11:12:57,523] INFO : classification_norm parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : regression_norm parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : classification_query parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank6 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank6 [2026-10-28 11:12:57,524] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-28 11:12:57,524] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_pid parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_pt parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_eta parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-28 11:12:57,524] INFO : nn_energy parameters: 0 +rank6 [2026-10-28 11:12:57,525] INFO : Total MLPF parameters: 7260244 +rank6 [2026-10-28 11:12:57,525] INFO : MLPF __init__ done +rank6 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank6 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=6 +rank6 [2026-10-28 11:12:57,558] INFO : Moved model to device rank=6 +rank6 [2026-10-28 11:12:57,560] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-28 11:12:57,560] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-28 11:12:57,812] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank6 [2026-10-28 11:12:57,812] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-28 11:12:57,813] INFO : Trainable parameters: 7260244 +rank6 [2026-10-28 11:12:57,813] INFO : Non-trainable parameters: 0 +rank6 [2026-10-28 11:12:57,813] INFO : Total parameters: 7260244 +rank6 [2026-10-28 11:12:57,816] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-28 11:12:57,816] INFO : Getting dataloaders +rank6 [2026-10-28 11:12:57,816] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:03,710] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:04,782] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,002] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,274] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,538] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,796] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-28 11:13:06,796] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-28 11:13:20,953] INFO : Got dataloaders +rank6 [2026-10-28 11:13:20,953] INFO : loader split=train rank=6 len=21108 +rank6 [2026-10-28 11:13:20,953] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-28 11:13:20,954] INFO : Starting training from step 1 to 50000 on rank 6 of 8 +rank6 [2026-10-28 11:13:20,954] INFO : Creating train_iterator +rank6 [2026-10-28 11:13:20,954] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-28 11:13:35,287] INFO : Created train_iterator +rank6 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank6 [2026-10-28 11:17:20,491] INFO : Step 100/50000 rank6 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7777 | Classification_binary: 0.5221 | Regression_cos_phi: 0.0072 | Regression_energy: 1.2778 | Regression_eta: 0.0159 | Regression_pt: 1.1991 | Regression_sin_phi: 0.0060 | LR: 1.00e-03 | DataLoad Time: 0.0151s | Model Forward Time: 3.1340s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank6 [2026-10-28 11:20:57,691] INFO : Step 200/50000 rank6 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4420 | Classification_binary: 0.5050 | Regression_cos_phi: 0.4715 | Regression_energy: 0.3514 | Regression_eta: 0.3919 | Regression_pt: 0.3751 | Regression_sin_phi: 0.4530 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 3.5611s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank6 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank6 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3281 | Classification_binary: 0.4826 | Regression_cos_phi: 0.3565 | Regression_energy: 0.3276 | Regression_eta: 0.3556 | Regression_pt: 0.3273 | Regression_sin_phi: 0.2967 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.7564s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank6 [2026-10-28 11:28:09,551] INFO : Step 400/50000 rank6 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2621 | Classification_binary: 0.4742 | Regression_cos_phi: 0.4008 | Regression_energy: 0.3287 | Regression_eta: 0.3537 | Regression_pt: 0.3366 | Regression_sin_phi: 0.3258 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.5808s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank6 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank6 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2879 | Classification_binary: 0.5041 | Regression_cos_phi: 0.3076 | Regression_energy: 0.2845 | Regression_eta: 0.3455 | Regression_pt: 0.3060 | Regression_sin_phi: 0.3949 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.0261s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank6 [2026-10-28 11:35:32,723] INFO : Step 600/50000 rank6 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2814 | Classification_binary: 0.4555 | Regression_cos_phi: 0.2646 | Regression_energy: 0.3084 | Regression_eta: 0.2527 | Regression_pt: 0.3282 | Regression_sin_phi: 0.3187 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.1154s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank6 [2026-10-28 11:39:09,492] INFO : Step 700/50000 rank6 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2313 | Classification_binary: 0.4179 | Regression_cos_phi: 0.2910 | Regression_energy: 0.2887 | Regression_eta: 0.2217 | Regression_pt: 0.3011 | Regression_sin_phi: 0.2866 | LR: 1.00e-03 | DataLoad Time: 0.0176s | Model Forward Time: 3.3985s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank6 [2026-10-28 11:42:49,057] INFO : Step 800/50000 rank6 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2704 | Classification_binary: 0.4757 | Regression_cos_phi: 0.3716 | Regression_energy: 0.2811 | Regression_eta: 0.3356 | Regression_pt: 0.2933 | Regression_sin_phi: 0.4420 | LR: 9.99e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.6508s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank6 [2026-10-28 11:46:31,921] INFO : Step 900/50000 rank6 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2715 | Classification_binary: 0.4876 | Regression_cos_phi: 0.2775 | Regression_energy: 0.2480 | Regression_eta: 0.2507 | Regression_pt: 0.2560 | Regression_sin_phi: 0.3033 | LR: 9.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5681s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank6 [2026-10-28 11:50:09,949] INFO : Step 1000/50000 rank6 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2203 | Classification_binary: 0.3883 | Regression_cos_phi: 0.1531 | Regression_energy: 0.2643 | Regression_eta: 0.1377 | Regression_pt: 0.2782 | Regression_sin_phi: 0.1894 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4166s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank6 [2026-10-28 11:50:10,911] INFO : Running validation on rank6 +rank6 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 11:50:58,529] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:50:59,086] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 11:51:19,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 11:51:20,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 11:51:21,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 11:51:22,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 11:51:32,505] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 11:51:32,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:51:33,312] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 11:51:48,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 11:51:49,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 11:51:50,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 11:51:51,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 11:52:01,753] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 11:52:02,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:52:03,474] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 11:52:18,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 11:52:19,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 11:52:20,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 11:52:20,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 11:52:30,869] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 11:56:43,807] INFO : Step 1100/50000 rank6 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2098 | Classification_binary: 0.4291 | Regression_cos_phi: 0.2131 | Regression_energy: 0.3241 | Regression_eta: 0.2223 | Regression_pt: 0.3294 | Regression_sin_phi: 0.2333 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9238s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank6 [2026-10-28 12:00:30,892] INFO : Step 1200/50000 rank6 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2220 | Classification_binary: 0.4982 | Regression_cos_phi: 0.2511 | Regression_energy: 0.2611 | Regression_eta: 0.2248 | Regression_pt: 0.2735 | Regression_sin_phi: 0.2681 | LR: 9.99e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4002s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank6 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank6 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1718 | Classification_binary: 0.4702 | Regression_cos_phi: 0.2709 | Regression_energy: 0.2180 | Regression_eta: 0.2472 | Regression_pt: 0.2293 | Regression_sin_phi: 0.2199 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1267s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank6 [2026-10-28 12:08:10,852] INFO : Step 1400/50000 rank6 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2486 | Classification_binary: 0.4105 | Regression_cos_phi: 0.2843 | Regression_energy: 0.2690 | Regression_eta: 0.2359 | Regression_pt: 0.2783 | Regression_sin_phi: 0.2681 | LR: 9.98e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7545s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank6 [2026-10-28 12:11:57,638] INFO : Step 1500/50000 rank6 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1998 | Classification_binary: 0.4391 | Regression_cos_phi: 0.2718 | Regression_energy: 0.2499 | Regression_eta: 0.2212 | Regression_pt: 0.2654 | Regression_sin_phi: 0.2928 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5811s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank6 [2026-10-28 12:15:43,594] INFO : Step 1600/50000 rank6 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1991 | Classification_binary: 0.4787 | Regression_cos_phi: 0.2326 | Regression_energy: 0.2601 | Regression_eta: 0.2402 | Regression_pt: 0.2721 | Regression_sin_phi: 0.2686 | LR: 9.98e-04 | DataLoad Time: 0.0241s | Model Forward Time: 4.0364s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank6 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank6 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1700 | Classification_binary: 0.3881 | Regression_cos_phi: 0.2141 | Regression_energy: 0.2776 | Regression_eta: 0.1783 | Regression_pt: 0.2875 | Regression_sin_phi: 0.1477 | LR: 9.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.3926s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank6 [2026-10-28 12:23:17,461] INFO : Step 1800/50000 rank6 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1958 | Classification_binary: 0.4123 | Regression_cos_phi: 0.1844 | Regression_energy: 0.2062 | Regression_eta: 0.1677 | Regression_pt: 0.2200 | Regression_sin_phi: 0.2439 | LR: 9.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9413s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank6 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank6 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2395 | Classification_binary: 0.5227 | Regression_cos_phi: 0.2744 | Regression_energy: 0.2813 | Regression_eta: 0.2432 | Regression_pt: 0.2943 | Regression_sin_phi: 0.2716 | LR: 9.97e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5701s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank6 [2026-10-28 12:30:59,046] INFO : Step 2000/50000 rank6 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1963 | Classification_binary: 0.3766 | Regression_cos_phi: 0.1844 | Regression_energy: 0.2153 | Regression_eta: 0.2022 | Regression_pt: 0.2247 | Regression_sin_phi: 0.2161 | LR: 9.96e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1419s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank6 [2026-10-28 12:31:00,064] INFO : Running validation on rank6 +rank6 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 12:31:47,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:31:48,295] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 12:32:13,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:32:14,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:32:15,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:32:16,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:32:26,624] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:32:27,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 12:32:44,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:32:45,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:32:46,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:32:47,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:32:57,025] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:32:57,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:32:57,641] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 12:33:13,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:33:14,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:33:14,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:33:15,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:33:25,505] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 12:37:32,606] INFO : Step 2100/50000 rank6 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2483 | Classification_binary: 0.4019 | Regression_cos_phi: 0.2223 | Regression_energy: 0.2540 | Regression_eta: 0.2439 | Regression_pt: 0.2668 | Regression_sin_phi: 0.2848 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7750s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank6 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank6 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2050 | Classification_binary: 0.4624 | Regression_cos_phi: 0.2816 | Regression_energy: 0.2787 | Regression_eta: 0.2265 | Regression_pt: 0.2923 | Regression_sin_phi: 0.2220 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7061s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank6 [2026-10-28 12:45:14,506] INFO : Step 2300/50000 rank6 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3537 | Regression_cos_phi: 0.1933 | Regression_energy: 0.2463 | Regression_eta: 0.1483 | Regression_pt: 0.2565 | Regression_sin_phi: 0.1716 | LR: 9.95e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.9854s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank6 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank6 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1901 | Classification_binary: 0.3562 | Regression_cos_phi: 0.2152 | Regression_energy: 0.2407 | Regression_eta: 0.2160 | Regression_pt: 0.2520 | Regression_sin_phi: 0.2217 | LR: 9.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4488s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank6 [2026-10-28 12:52:58,131] INFO : Step 2500/50000 rank6 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2345 | Classification_binary: 0.4322 | Regression_cos_phi: 0.2422 | Regression_energy: 0.2500 | Regression_eta: 0.2212 | Regression_pt: 0.2650 | Regression_sin_phi: 0.2911 | LR: 9.94e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3988s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank6 [2026-10-28 12:56:45,817] INFO : Step 2600/50000 rank6 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1608 | Classification_binary: 0.4265 | Regression_cos_phi: 0.2002 | Regression_energy: 0.2496 | Regression_eta: 0.2060 | Regression_pt: 0.2613 | Regression_sin_phi: 0.2291 | LR: 9.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank6 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank6 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1681 | Classification_binary: 0.3184 | Regression_cos_phi: 0.2435 | Regression_energy: 0.2216 | Regression_eta: 0.2069 | Regression_pt: 0.2314 | Regression_sin_phi: 0.1951 | LR: 9.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0889s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank6 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank6 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1955 | Classification_binary: 0.3891 | Regression_cos_phi: 0.2127 | Regression_energy: 0.2210 | Regression_eta: 0.1926 | Regression_pt: 0.2341 | Regression_sin_phi: 0.2010 | LR: 9.93e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank6 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank6 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2125 | Classification_binary: 0.4264 | Regression_cos_phi: 0.2726 | Regression_energy: 0.2525 | Regression_eta: 0.2793 | Regression_pt: 0.2624 | Regression_sin_phi: 0.2798 | LR: 9.93e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1926s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank6 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank6 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1891 | Classification_binary: 0.4076 | Regression_cos_phi: 0.2325 | Regression_energy: 0.2228 | Regression_eta: 0.2786 | Regression_pt: 0.2351 | Regression_sin_phi: 0.2779 | LR: 9.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5278s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank6 [2026-10-28 13:12:10,495] INFO : Running validation on rank6 +rank6 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:13:02,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:13:03,432] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 13:13:30,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:13:31,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:13:32,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:13:33,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:13:43,177] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:13:44,105] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 13:14:00,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:14:01,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:14:02,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:14:03,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:14:13,370] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:14:13,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:14:14,292] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 13:14:29,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:14:30,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:14:31,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:14:31,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:14:41,822] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 13:19:02,741] INFO : Step 3100/50000 rank6 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1725 | Classification_binary: 0.3344 | Regression_cos_phi: 0.1961 | Regression_energy: 0.2298 | Regression_eta: 0.1965 | Regression_pt: 0.2506 | Regression_sin_phi: 0.2694 | LR: 9.91e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.5847s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank6 [2026-10-28 13:23:00,673] INFO : Step 3200/50000 rank6 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1920 | Classification_binary: 0.4113 | Regression_cos_phi: 0.1719 | Regression_energy: 0.2430 | Regression_eta: 0.2122 | Regression_pt: 0.2554 | Regression_sin_phi: 0.2146 | LR: 9.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8520s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank6 [2026-10-28 13:26:54,112] INFO : Step 3300/50000 rank6 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1834 | Classification_binary: 0.3927 | Regression_cos_phi: 0.1941 | Regression_energy: 0.2204 | Regression_eta: 0.1915 | Regression_pt: 0.2268 | Regression_sin_phi: 0.2382 | LR: 9.90e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7306s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank6 [2026-10-28 13:30:48,599] INFO : Step 3400/50000 rank6 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1517 | Classification_binary: 0.3452 | Regression_cos_phi: 0.2224 | Regression_energy: 0.2403 | Regression_eta: 0.1397 | Regression_pt: 0.2478 | Regression_sin_phi: 0.1943 | LR: 9.90e-04 | DataLoad Time: 0.0128s | Model Forward Time: 4.0703s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank6 [2026-10-28 13:34:47,771] INFO : Step 3500/50000 rank6 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1822 | Classification_binary: 0.3856 | Regression_cos_phi: 0.2367 | Regression_energy: 0.2294 | Regression_eta: 0.1704 | Regression_pt: 0.2429 | Regression_sin_phi: 0.1879 | LR: 9.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6675s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank6 [2026-10-28 13:38:46,122] INFO : Step 3600/50000 rank6 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1482 | Classification_binary: 0.4472 | Regression_cos_phi: 0.2139 | Regression_energy: 0.2116 | Regression_eta: 0.1846 | Regression_pt: 0.2228 | Regression_sin_phi: 0.2094 | LR: 9.89e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.1030s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank6 [2026-10-28 13:42:35,793] INFO : Step 3700/50000 rank6 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1575 | Classification_binary: 0.3183 | Regression_cos_phi: 0.2277 | Regression_energy: 0.1987 | Regression_eta: 0.2157 | Regression_pt: 0.2046 | Regression_sin_phi: 0.2008 | LR: 9.88e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2039s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank6 [2026-10-28 13:46:34,311] INFO : Step 3800/50000 rank6 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1596 | Classification_binary: 0.3196 | Regression_cos_phi: 0.1703 | Regression_energy: 0.2080 | Regression_eta: 0.1364 | Regression_pt: 0.2179 | Regression_sin_phi: 0.2044 | LR: 9.87e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9866s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank6 [2026-10-28 13:50:23,156] INFO : Step 3900/50000 rank6 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1783 | Classification_binary: 0.4471 | Regression_cos_phi: 0.2252 | Regression_energy: 0.2086 | Regression_eta: 0.2132 | Regression_pt: 0.2193 | Regression_sin_phi: 0.2591 | LR: 9.87e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2428s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank6 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank6 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1510 | Classification_binary: 0.3368 | Regression_cos_phi: 0.1852 | Regression_energy: 0.2034 | Regression_eta: 0.1510 | Regression_pt: 0.2113 | Regression_sin_phi: 0.2211 | LR: 9.86e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5791s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank6 [2026-10-28 13:54:12,844] INFO : Running validation on rank6 +rank6 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:55:03,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 13:55:33,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:55:34,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:55:35,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:55:36,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:55:46,149] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:55:46,898] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 13:56:02,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:56:03,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:56:04,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:56:05,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:56:15,599] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:56:16,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:56:16,986] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 13:56:32,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:56:32,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:56:33,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:56:34,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:56:44,127] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank6 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1805 | Classification_binary: 0.3752 | Regression_cos_phi: 0.2710 | Regression_energy: 0.2500 | Regression_eta: 0.2043 | Regression_pt: 0.2575 | Regression_sin_phi: 0.2476 | LR: 9.85e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9541s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank6 [2026-10-28 14:04:50,729] INFO : Step 4200/50000 rank6 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1878 | Classification_binary: 0.4126 | Regression_cos_phi: 0.2329 | Regression_energy: 0.2209 | Regression_eta: 0.2561 | Regression_pt: 0.2302 | Regression_sin_phi: 0.2598 | LR: 9.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4857s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank6 [2026-10-28 14:08:42,269] INFO : Step 4300/50000 rank6 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1388 | Classification_binary: 0.3560 | Regression_cos_phi: 0.2092 | Regression_energy: 0.1992 | Regression_eta: 0.1616 | Regression_pt: 0.2086 | Regression_sin_phi: 0.1638 | LR: 9.84e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.1044s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank6 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank6 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1279 | Classification_binary: 0.2859 | Regression_cos_phi: 0.1301 | Regression_energy: 0.2121 | Regression_eta: 0.1513 | Regression_pt: 0.2183 | Regression_sin_phi: 0.1637 | LR: 9.83e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7775s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank6 [2026-10-28 14:16:26,827] INFO : Step 4500/50000 rank6 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2015 | Classification_binary: 0.3943 | Regression_cos_phi: 0.3293 | Regression_energy: 0.2273 | Regression_eta: 0.2371 | Regression_pt: 0.2368 | Regression_sin_phi: 0.2472 | LR: 9.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4684s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank6 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank6 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1729 | Classification_binary: 0.4332 | Regression_cos_phi: 0.1892 | Regression_energy: 0.2127 | Regression_eta: 0.2215 | Regression_pt: 0.2210 | Regression_sin_phi: 0.2106 | LR: 9.81e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1356s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank6 [2026-10-28 14:24:22,132] INFO : Step 4700/50000 rank6 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1612 | Classification_binary: 0.3260 | Regression_cos_phi: 0.1984 | Regression_energy: 0.1971 | Regression_eta: 0.1757 | Regression_pt: 0.2075 | Regression_sin_phi: 0.2074 | LR: 9.81e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.2978s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank6 [2026-10-28 14:28:09,270] INFO : Step 4800/50000 rank6 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1415 | Classification_binary: 0.3396 | Regression_cos_phi: 0.2268 | Regression_energy: 0.2015 | Regression_eta: 0.1682 | Regression_pt: 0.2126 | Regression_sin_phi: 0.2336 | LR: 9.80e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9653s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank6 [2026-10-28 14:32:01,139] INFO : Step 4900/50000 rank6 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1930 | Classification_binary: 0.4027 | Regression_cos_phi: 0.2512 | Regression_energy: 0.2173 | Regression_eta: 0.2468 | Regression_pt: 0.2286 | Regression_sin_phi: 0.2416 | LR: 9.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8455s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank6 [2026-10-28 14:35:52,796] INFO : Step 5000/50000 rank6 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1651 | Classification_binary: 0.3708 | Regression_cos_phi: 0.1806 | Regression_energy: 0.1845 | Regression_eta: 0.1732 | Regression_pt: 0.1949 | Regression_sin_phi: 0.2232 | LR: 9.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8400s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank6 [2026-10-28 14:35:53,776] INFO : Running validation on rank6 +rank6 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 14:36:42,157] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:36:42,706] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 14:37:08,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 14:37:10,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 14:37:11,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 14:37:12,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 14:37:22,344] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:37:22,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 14:37:38,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 14:37:39,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 14:37:40,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 14:37:41,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 14:37:51,747] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 14:37:52,214] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:37:52,790] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 14:38:08,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 14:38:08,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 14:38:09,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 14:38:10,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 14:38:20,180] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank6 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1601 | Classification_binary: 0.3215 | Regression_cos_phi: 0.2717 | Regression_energy: 0.1801 | Regression_eta: 0.1907 | Regression_pt: 0.1873 | Regression_sin_phi: 0.1687 | LR: 9.77e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3402s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank6 [2026-10-28 14:46:36,780] INFO : Step 5200/50000 rank6 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1627 | Classification_binary: 0.3809 | Regression_cos_phi: 0.2207 | Regression_energy: 0.2050 | Regression_eta: 0.1843 | Regression_pt: 0.2157 | Regression_sin_phi: 0.2071 | LR: 9.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9483s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank6 [2026-10-28 14:50:33,436] INFO : Step 5300/50000 rank6 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1500 | Classification_binary: 0.3745 | Regression_cos_phi: 0.2028 | Regression_energy: 0.2026 | Regression_eta: 0.1498 | Regression_pt: 0.2154 | Regression_sin_phi: 0.1806 | LR: 9.75e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.1857s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank6 [2026-10-28 14:54:36,492] INFO : Step 5400/50000 rank6 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2821 | Regression_cos_phi: 0.2323 | Regression_energy: 0.1952 | Regression_eta: 0.1239 | Regression_pt: 0.2006 | Regression_sin_phi: 0.1524 | LR: 9.74e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.4924s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank6 [2026-10-28 14:58:42,457] INFO : Step 5500/50000 rank6 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1321 | Classification_binary: 0.2878 | Regression_cos_phi: 0.1841 | Regression_energy: 0.2132 | Regression_eta: 0.1443 | Regression_pt: 0.2227 | Regression_sin_phi: 0.1629 | LR: 9.73e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.5903s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank6 [2026-10-28 15:02:44,103] INFO : Step 5600/50000 rank6 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1760 | Classification_binary: 0.4139 | Regression_cos_phi: 0.2001 | Regression_energy: 0.1851 | Regression_eta: 0.1869 | Regression_pt: 0.1915 | Regression_sin_phi: 0.2056 | LR: 9.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6386s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank6 [2026-10-28 15:06:32,821] INFO : Step 5700/50000 rank6 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1209 | Classification_binary: 0.3072 | Regression_cos_phi: 0.1965 | Regression_energy: 0.1952 | Regression_eta: 0.1858 | Regression_pt: 0.2069 | Regression_sin_phi: 0.1541 | LR: 9.71e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1530s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank6 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank6 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1081 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1301 | Regression_energy: 0.1928 | Regression_eta: 0.1161 | Regression_pt: 0.1994 | Regression_sin_phi: 0.1684 | LR: 9.70e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5289s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank6 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank6 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1605 | Classification_binary: 0.3375 | Regression_cos_phi: 0.1971 | Regression_energy: 0.1875 | Regression_eta: 0.2298 | Regression_pt: 0.1974 | Regression_sin_phi: 0.2568 | LR: 9.69e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.0729s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank6 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank6 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1568 | Classification_binary: 0.3562 | Regression_cos_phi: 0.1860 | Regression_energy: 0.1796 | Regression_eta: 0.1379 | Regression_pt: 0.1883 | Regression_sin_phi: 0.2105 | LR: 9.68e-04 | DataLoad Time: 0.0114s | Model Forward Time: 6.3879s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank6 [2026-10-28 15:18:46,644] INFO : Running validation on rank6 +rank6 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 15:19:34,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:19:35,275] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 15:19:55,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 15:19:57,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 15:19:58,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 15:20:00,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 15:20:10,498] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:20:11,688] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 15:20:27,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 15:20:28,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 15:20:29,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 15:20:30,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 15:20:40,801] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:20:40,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:20:41,483] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 15:20:57,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 15:20:58,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 15:20:58,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 15:20:59,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 15:21:09,630] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 15:25:37,242] INFO : Step 6100/50000 rank6 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1105 | Classification_binary: 0.2591 | Regression_cos_phi: 0.1451 | Regression_energy: 0.1612 | Regression_eta: 0.2194 | Regression_pt: 0.1679 | Regression_sin_phi: 0.1517 | LR: 9.67e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2765s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank6 [2026-10-28 15:29:27,138] INFO : Step 6200/50000 rank6 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 0.3528 | Regression_cos_phi: 0.1938 | Regression_energy: 0.1924 | Regression_eta: 0.2232 | Regression_pt: 0.2037 | Regression_sin_phi: 0.2618 | LR: 9.66e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.3985s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank6 [2026-10-28 15:33:16,837] INFO : Step 6300/50000 rank6 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1683 | Classification_binary: 0.4117 | Regression_cos_phi: 0.2079 | Regression_energy: 0.1696 | Regression_eta: 0.1982 | Regression_pt: 0.1809 | Regression_sin_phi: 0.2089 | LR: 9.65e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2464s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank6 [2026-10-28 15:37:09,469] INFO : Step 6400/50000 rank6 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.2803 | Regression_cos_phi: 0.1535 | Regression_energy: 0.1841 | Regression_eta: 0.1243 | Regression_pt: 0.1912 | Regression_sin_phi: 0.1063 | LR: 9.64e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9966s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank6 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank6 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.3561 | Regression_cos_phi: 0.2089 | Regression_energy: 0.1766 | Regression_eta: 0.1764 | Regression_pt: 0.1874 | Regression_sin_phi: 0.2156 | LR: 9.63e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6705s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank6 [2026-10-28 15:45:09,041] INFO : Step 6600/50000 rank6 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1435 | Classification_binary: 0.3841 | Regression_cos_phi: 0.1527 | Regression_energy: 0.1759 | Regression_eta: 0.1211 | Regression_pt: 0.1816 | Regression_sin_phi: 0.1918 | LR: 9.62e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2532s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank6 [2026-10-28 15:49:06,067] INFO : Step 6700/50000 rank6 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1135 | Classification_binary: 0.3049 | Regression_cos_phi: 0.1698 | Regression_energy: 0.1737 | Regression_eta: 0.1748 | Regression_pt: 0.1789 | Regression_sin_phi: 0.1541 | LR: 9.61e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.0300s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank6 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank6 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1188 | Classification_binary: 0.2761 | Regression_cos_phi: 0.1394 | Regression_energy: 0.1869 | Regression_eta: 0.1640 | Regression_pt: 0.1959 | Regression_sin_phi: 0.1505 | LR: 9.60e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8861s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank6 [2026-10-28 15:56:38,729] INFO : Step 6900/50000 rank6 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1542 | Classification_binary: 0.3655 | Regression_cos_phi: 0.2404 | Regression_energy: 0.1899 | Regression_eta: 0.2141 | Regression_pt: 0.2018 | Regression_sin_phi: 0.2165 | LR: 9.58e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.8839s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank6 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank6 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1483 | Classification_binary: 0.3979 | Regression_cos_phi: 0.3143 | Regression_energy: 0.1629 | Regression_eta: 0.1968 | Regression_pt: 0.1688 | Regression_sin_phi: 0.2911 | LR: 9.57e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4723s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank6 [2026-10-28 16:00:38,672] INFO : Running validation on rank6 +rank6 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:01:32,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:01:33,700] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 16:02:00,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:02:01,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:02:02,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:02:03,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:02:13,265] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:02:14,069] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 16:02:30,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:02:31,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:02:32,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:02:33,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:02:43,425] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:02:44,278] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 16:02:59,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:03:00,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:03:01,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:03:01,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:03:11,979] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 16:07:39,194] INFO : Step 7100/50000 rank6 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2474 | Regression_cos_phi: 0.1400 | Regression_energy: 0.1905 | Regression_eta: 0.1430 | Regression_pt: 0.1979 | Regression_sin_phi: 0.1286 | LR: 9.56e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5615s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank6 [2026-10-28 16:11:41,309] INFO : Step 7200/50000 rank6 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1588 | Classification_binary: 0.3204 | Regression_cos_phi: 0.3224 | Regression_energy: 0.1621 | Regression_eta: 0.1550 | Regression_pt: 0.1700 | Regression_sin_phi: 0.1736 | LR: 9.55e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9563s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank6 [2026-10-28 16:15:45,432] INFO : Step 7300/50000 rank6 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1353 | Classification_binary: 0.3628 | Regression_cos_phi: 0.2022 | Regression_energy: 0.1605 | Regression_eta: 0.1361 | Regression_pt: 0.1649 | Regression_sin_phi: 0.1869 | LR: 9.53e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.8975s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank6 [2026-10-28 16:19:47,791] INFO : Step 7400/50000 rank6 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1019 | Classification_binary: 0.2851 | Regression_cos_phi: 0.1384 | Regression_energy: 0.1839 | Regression_eta: 0.1198 | Regression_pt: 0.1921 | Regression_sin_phi: 0.1437 | LR: 9.52e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.6024s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank6 [2026-10-28 16:23:56,336] INFO : Step 7500/50000 rank6 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1229 | Classification_binary: 0.2460 | Regression_cos_phi: 0.1784 | Regression_energy: 0.1687 | Regression_eta: 0.1289 | Regression_pt: 0.1754 | Regression_sin_phi: 0.1945 | LR: 9.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7917s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank6 [2026-10-28 16:27:58,400] INFO : Step 7600/50000 rank6 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1672 | Classification_binary: 0.3421 | Regression_cos_phi: 0.1826 | Regression_energy: 0.1680 | Regression_eta: 0.1902 | Regression_pt: 0.1733 | Regression_sin_phi: 0.2298 | LR: 9.50e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6996s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank6 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank6 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1310 | Classification_binary: 0.3286 | Regression_cos_phi: 0.1689 | Regression_energy: 0.1342 | Regression_eta: 0.1424 | Regression_pt: 0.1382 | Regression_sin_phi: 0.1912 | LR: 9.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9797s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank6 [2026-10-28 16:35:55,393] INFO : Step 7800/50000 rank6 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0922 | Classification_binary: 0.2702 | Regression_cos_phi: 0.1751 | Regression_energy: 0.1724 | Regression_eta: 0.1296 | Regression_pt: 0.1795 | Regression_sin_phi: 0.1115 | LR: 9.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2110s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank6 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank6 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1332 | Classification_binary: 0.3403 | Regression_cos_phi: 0.1658 | Regression_energy: 0.1729 | Regression_eta: 0.1464 | Regression_pt: 0.1802 | Regression_sin_phi: 0.2044 | LR: 9.46e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3422s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank6 [2026-10-28 16:44:12,790] INFO : Step 8000/50000 rank6 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1511 | Classification_binary: 0.3441 | Regression_cos_phi: 0.1845 | Regression_energy: 0.1569 | Regression_eta: 0.1545 | Regression_pt: 0.1629 | Regression_sin_phi: 0.1704 | LR: 9.44e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2452s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank6 [2026-10-28 16:44:13,936] INFO : Running validation on rank6 +rank6 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:45:04,006] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:45:04,562] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 16:45:30,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:45:31,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:45:32,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:45:34,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:45:44,277] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:45:45,752] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 16:46:01,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:46:02,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:46:03,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:46:04,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:46:14,413] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:46:15,125] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 16:46:30,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:46:31,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:46:32,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:46:32,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:46:42,808] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank6 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2838 | Regression_cos_phi: 0.1215 | Regression_energy: 0.1670 | Regression_eta: 0.1254 | Regression_pt: 0.1766 | Regression_sin_phi: 0.1128 | LR: 9.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7763s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank6 [2026-10-28 16:55:30,391] INFO : Step 8200/50000 rank6 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1789 | Regression_energy: 0.1448 | Regression_eta: 0.1565 | Regression_pt: 0.1534 | Regression_sin_phi: 0.1870 | LR: 9.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1551s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank6 [2026-10-28 16:59:31,718] INFO : Step 8300/50000 rank6 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.3408 | Regression_cos_phi: 0.1751 | Regression_energy: 0.1514 | Regression_eta: 0.1522 | Regression_pt: 0.1594 | Regression_sin_phi: 0.1382 | LR: 9.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.9183s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank6 [2026-10-28 17:03:26,315] INFO : Step 8400/50000 rank6 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1176 | Classification_binary: 0.2628 | Regression_cos_phi: 0.1464 | Regression_energy: 0.1473 | Regression_eta: 0.1217 | Regression_pt: 0.1526 | Regression_sin_phi: 0.1368 | LR: 9.39e-04 | DataLoad Time: 0.0232s | Model Forward Time: 6.0436s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank6 [2026-10-28 17:07:18,317] INFO : Step 8500/50000 rank6 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.2496 | Regression_cos_phi: 0.1219 | Regression_energy: 0.1294 | Regression_eta: 0.1295 | Regression_pt: 0.1321 | Regression_sin_phi: 0.1444 | LR: 9.37e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.4974s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank6 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank6 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1255 | Classification_binary: 0.3214 | Regression_cos_phi: 0.1339 | Regression_energy: 0.1315 | Regression_eta: 0.1149 | Regression_pt: 0.1360 | Regression_sin_phi: 0.1586 | LR: 9.36e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1710s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank6 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank6 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2949 | Regression_cos_phi: 0.1731 | Regression_energy: 0.1505 | Regression_eta: 0.1304 | Regression_pt: 0.1557 | Regression_sin_phi: 0.1240 | LR: 9.34e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.2143s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank6 [2026-10-28 17:19:40,247] INFO : Step 8800/50000 rank6 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.2770 | Regression_cos_phi: 0.1641 | Regression_energy: 0.1370 | Regression_eta: 0.1355 | Regression_pt: 0.1430 | Regression_sin_phi: 0.1182 | LR: 9.33e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.4421s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank6 [2026-10-28 17:23:38,392] INFO : Step 8900/50000 rank6 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.3585 | Regression_cos_phi: 0.1708 | Regression_energy: 0.1617 | Regression_eta: 0.1398 | Regression_pt: 0.1682 | Regression_sin_phi: 0.1668 | LR: 9.31e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6549s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank6 [2026-10-28 17:27:33,262] INFO : Step 9000/50000 rank6 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1329 | Classification_binary: 0.3079 | Regression_cos_phi: 0.1616 | Regression_energy: 0.1587 | Regression_eta: 0.1415 | Regression_pt: 0.1678 | Regression_sin_phi: 0.1671 | LR: 9.30e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank6 [2026-10-28 17:27:34,333] INFO : Running validation on rank6 +rank6 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 17:28:24,726] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:28:25,265] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 17:28:52,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 17:28:53,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 17:28:54,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 17:28:55,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 17:29:05,197] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:29:06,208] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 17:29:22,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 17:29:23,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 17:29:25,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 17:29:26,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 17:29:36,219] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 17:29:37,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:29:37,692] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 17:29:53,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 17:29:54,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 17:29:54,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 17:29:55,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 17:30:05,465] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 17:34:23,897] INFO : Step 9100/50000 rank6 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.2356 | Regression_cos_phi: 0.1492 | Regression_energy: 0.1390 | Regression_eta: 0.1071 | Regression_pt: 0.1434 | Regression_sin_phi: 0.1188 | LR: 9.28e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1034s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank6 [2026-10-28 17:38:24,814] INFO : Step 9200/50000 rank6 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3076 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1595 | Regression_eta: 0.1499 | Regression_pt: 0.1648 | Regression_sin_phi: 0.1832 | LR: 9.27e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3266s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank6 [2026-10-28 17:42:44,641] INFO : Step 9300/50000 rank6 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1426 | Classification_binary: 0.3830 | Regression_cos_phi: 0.2093 | Regression_energy: 0.1783 | Regression_eta: 0.1584 | Regression_pt: 0.1887 | Regression_sin_phi: 0.1583 | LR: 9.25e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.8262s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank6 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank6 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2660 | Regression_cos_phi: 0.1484 | Regression_energy: 0.1266 | Regression_eta: 0.1358 | Regression_pt: 0.1330 | Regression_sin_phi: 0.1340 | LR: 9.24e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4119s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank6 [2026-10-28 17:51:11,694] INFO : Step 9500/50000 rank6 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2773 | Regression_cos_phi: 0.1730 | Regression_energy: 0.1448 | Regression_eta: 0.1703 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1854 | LR: 9.22e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.2546s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank6 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank6 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.3093 | Regression_cos_phi: 0.1825 | Regression_energy: 0.1452 | Regression_eta: 0.1297 | Regression_pt: 0.1521 | Regression_sin_phi: 0.1751 | LR: 9.21e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.8392s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank6 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank6 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.3017 | Regression_cos_phi: 0.1232 | Regression_energy: 0.1372 | Regression_eta: 0.0920 | Regression_pt: 0.1444 | Regression_sin_phi: 0.1364 | LR: 9.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9948s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank6 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank6 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2845 | Regression_cos_phi: 0.1210 | Regression_energy: 0.1303 | Regression_eta: 0.1100 | Regression_pt: 0.1398 | Regression_sin_phi: 0.1372 | LR: 9.17e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6867s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank6 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank6 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2693 | Regression_cos_phi: 0.1299 | Regression_energy: 0.1316 | Regression_eta: 0.0885 | Regression_pt: 0.1409 | Regression_sin_phi: 0.1094 | LR: 9.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1765s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank6 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank6 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.3311 | Regression_cos_phi: 0.1430 | Regression_energy: 0.1371 | Regression_eta: 0.1529 | Regression_pt: 0.1433 | Regression_sin_phi: 0.1667 | LR: 9.14e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1268s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank6 [2026-10-28 18:11:36,667] INFO : Running validation on rank6 +rank6 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:12:31,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:12:31,989] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 18:12:58,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:12:59,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:13:00,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:13:01,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:13:11,360] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:13:12,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:13:12,669] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 18:13:29,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:13:30,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:13:31,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:13:32,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:13:42,387] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:13:48,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:13:49,238] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 18:14:04,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:14:05,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:14:05,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:14:06,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:14:16,490] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 18:18:32,573] INFO : Step 10100/50000 rank6 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0818 | Classification_binary: 0.2612 | Regression_cos_phi: 0.1170 | Regression_energy: 0.1256 | Regression_eta: 0.1224 | Regression_pt: 0.1310 | Regression_sin_phi: 0.1288 | LR: 9.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.2348s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank6 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank6 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1279 | Classification_binary: 0.2514 | Regression_cos_phi: 0.1871 | Regression_energy: 0.1309 | Regression_eta: 0.1453 | Regression_pt: 0.1347 | Regression_sin_phi: 0.1406 | LR: 9.11e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5663s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank6 [2026-10-28 18:26:52,297] INFO : Step 10300/50000 rank6 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.3272 | Regression_cos_phi: 0.1213 | Regression_energy: 0.1228 | Regression_eta: 0.0893 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0972 | LR: 9.09e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.3472s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank6 [2026-10-28 18:31:09,156] INFO : Step 10400/50000 rank6 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0915 | Classification_binary: 0.2891 | Regression_cos_phi: 0.1205 | Regression_energy: 0.1272 | Regression_eta: 0.1126 | Regression_pt: 0.1323 | Regression_sin_phi: 0.1249 | LR: 9.07e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9055s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank6 [2026-10-28 18:35:25,946] INFO : Step 10500/50000 rank6 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0981 | Regression_energy: 0.1349 | Regression_eta: 0.0727 | Regression_pt: 0.1397 | Regression_sin_phi: 0.1004 | LR: 9.06e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.8297s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank6 [2026-10-28 18:39:26,894] INFO : Step 10600/50000 rank6 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1350 | Regression_energy: 0.1289 | Regression_eta: 0.0933 | Regression_pt: 0.1343 | Regression_sin_phi: 0.1039 | LR: 9.04e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2170s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank6 [2026-10-28 18:43:27,488] INFO : Step 10700/50000 rank6 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0981 | Classification_binary: 0.3237 | Regression_cos_phi: 0.0988 | Regression_energy: 0.1341 | Regression_eta: 0.1086 | Regression_pt: 0.1379 | Regression_sin_phi: 0.1464 | LR: 9.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2747s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank6 [2026-10-28 18:47:34,478] INFO : Step 10800/50000 rank6 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2337 | Regression_cos_phi: 0.1044 | Regression_energy: 0.1334 | Regression_eta: 0.1101 | Regression_pt: 0.1396 | Regression_sin_phi: 0.1236 | LR: 9.00e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.3721s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank6 [2026-10-28 18:51:51,744] INFO : Step 10900/50000 rank6 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2495 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1188 | Regression_eta: 0.0729 | Regression_pt: 0.1280 | Regression_sin_phi: 0.1258 | LR: 8.99e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.6007s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank6 [2026-10-28 18:55:59,247] INFO : Step 11000/50000 rank6 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0894 | Classification_binary: 0.2972 | Regression_cos_phi: 0.1007 | Regression_energy: 0.1383 | Regression_eta: 0.0794 | Regression_pt: 0.1419 | Regression_sin_phi: 0.1020 | LR: 8.97e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.8642s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank6 [2026-10-28 18:56:00,217] INFO : Running validation on rank6 +rank6 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:56:54,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:56:55,374] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 18:57:22,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:57:23,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:57:23,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:57:25,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:57:35,074] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:57:35,272] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:57:35,835] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 18:57:51,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:57:52,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:57:54,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:57:55,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:58:05,045] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:58:07,004] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 18:58:22,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:58:23,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:58:24,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:58:24,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:58:34,718] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 19:03:04,099] INFO : Step 11100/50000 rank6 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.2803 | Regression_cos_phi: 0.0929 | Regression_energy: 0.1115 | Regression_eta: 0.0855 | Regression_pt: 0.1147 | Regression_sin_phi: 0.1159 | LR: 8.95e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.4739s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank6 [2026-10-28 19:07:10,330] INFO : Step 11200/50000 rank6 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2470 | Regression_cos_phi: 0.1151 | Regression_energy: 0.1184 | Regression_eta: 0.1362 | Regression_pt: 0.1235 | Regression_sin_phi: 0.1253 | LR: 8.93e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8093s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank6 [2026-10-28 19:11:04,857] INFO : Step 11300/50000 rank6 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2782 | Regression_cos_phi: 0.0970 | Regression_energy: 0.1348 | Regression_eta: 0.0797 | Regression_pt: 0.1431 | Regression_sin_phi: 0.1078 | LR: 8.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2489s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank6 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank6 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2698 | Regression_cos_phi: 0.1135 | Regression_energy: 0.1263 | Regression_eta: 0.1450 | Regression_pt: 0.1323 | Regression_sin_phi: 0.1169 | LR: 8.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0019s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank6 [2026-10-28 19:18:58,942] INFO : Step 11500/50000 rank6 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2075 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1092 | Regression_eta: 0.0703 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0970 | LR: 8.88e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.4532s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank6 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank6 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2798 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1247 | Regression_eta: 0.1033 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0982 | LR: 8.86e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8309s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank6 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank6 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2919 | Regression_cos_phi: 0.0873 | Regression_energy: 0.1171 | Regression_eta: 0.0990 | Regression_pt: 0.1213 | Regression_sin_phi: 0.1065 | LR: 8.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1788s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank6 [2026-10-28 19:30:54,032] INFO : Step 11800/50000 rank6 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2067 | Regression_cos_phi: 0.1742 | Regression_energy: 0.1083 | Regression_eta: 0.1057 | Regression_pt: 0.1122 | Regression_sin_phi: 0.1109 | LR: 8.82e-04 | DataLoad Time: 0.0254s | Model Forward Time: 6.0733s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank6 [2026-10-28 19:34:55,960] INFO : Step 11900/50000 rank6 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2638 | Regression_cos_phi: 0.1376 | Regression_energy: 0.1244 | Regression_eta: 0.0977 | Regression_pt: 0.1296 | Regression_sin_phi: 0.1258 | LR: 8.80e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9519s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank6 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank6 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.3117 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1112 | Regression_eta: 0.0742 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0951 | LR: 8.78e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.3563s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank6 [2026-10-28 19:39:04,976] INFO : Running validation on rank6 +rank6 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 19:39:55,050] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:39:55,582] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 19:40:21,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 19:40:22,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 19:40:23,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 19:40:24,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 19:40:34,537] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 19:40:34,842] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:40:35,410] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 19:40:51,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 19:40:52,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 19:40:53,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 19:40:54,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 19:41:04,406] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 19:41:05,105] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:41:05,608] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 19:41:21,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 19:41:21,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 19:41:22,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 19:41:23,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 19:41:33,296] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 19:45:57,644] INFO : Step 12100/50000 rank6 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2245 | Regression_cos_phi: 0.0934 | Regression_energy: 0.1096 | Regression_eta: 0.0726 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0882 | LR: 8.76e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8584s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank6 [2026-10-28 19:50:02,390] INFO : Step 12200/50000 rank6 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1051 | Regression_eta: 0.0473 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0816 | LR: 8.74e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.9605s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank6 [2026-10-28 19:54:00,333] INFO : Step 12300/50000 rank6 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1066 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1430 | Regression_energy: 0.1166 | Regression_eta: 0.1115 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1443 | LR: 8.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4598s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank6 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank6 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2779 | Regression_cos_phi: 0.1058 | Regression_energy: 0.1054 | Regression_eta: 0.1195 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1384 | LR: 8.70e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9479s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank6 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank6 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0643 | Regression_energy: 0.1157 | Regression_eta: 0.0572 | Regression_pt: 0.1189 | Regression_sin_phi: 0.1273 | LR: 8.68e-04 | DataLoad Time: 0.0239s | Model Forward Time: 4.6928s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank6 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank6 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2852 | Regression_cos_phi: 0.1228 | Regression_energy: 0.1137 | Regression_eta: 0.0866 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1133 | LR: 8.66e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9516s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank6 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank6 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.3134 | Regression_cos_phi: 0.1154 | Regression_energy: 0.1090 | Regression_eta: 0.0945 | Regression_pt: 0.1150 | Regression_sin_phi: 0.1072 | LR: 8.64e-04 | DataLoad Time: 0.0137s | Model Forward Time: 5.2260s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank6 [2026-10-28 20:14:52,957] INFO : Step 12800/50000 rank6 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.2489 | Regression_cos_phi: 0.1029 | Regression_energy: 0.1054 | Regression_eta: 0.0788 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0930 | LR: 8.62e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.8533s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank6 [2026-10-28 20:19:01,949] INFO : Step 12900/50000 rank6 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2730 | Regression_cos_phi: 0.1269 | Regression_energy: 0.1138 | Regression_eta: 0.1363 | Regression_pt: 0.1178 | Regression_sin_phi: 0.1529 | LR: 8.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5205s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank6 [2026-10-28 20:23:25,504] INFO : Step 13000/50000 rank6 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.3321 | Regression_cos_phi: 0.1760 | Regression_energy: 0.0953 | Regression_eta: 0.1346 | Regression_pt: 0.1009 | Regression_sin_phi: 0.1410 | LR: 8.58e-04 | DataLoad Time: 0.0158s | Model Forward Time: 10.3376s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank6 [2026-10-28 20:23:26,442] INFO : Running validation on rank6 +rank6 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 20:24:18,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 20:24:45,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 20:24:46,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 20:24:47,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 20:24:48,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 20:24:58,306] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 20:25:04,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:25:04,756] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 20:25:20,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 20:25:21,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 20:25:22,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 20:25:23,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 20:25:33,673] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:25:34,726] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 20:25:50,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 20:25:51,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 20:25:52,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 20:25:52,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 20:26:02,871] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 20:30:28,057] INFO : Step 13100/50000 rank6 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2221 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0754 | Regression_eta: 0.0859 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0723 | LR: 8.56e-04 | DataLoad Time: 0.0228s | Model Forward Time: 5.0190s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank6 [2026-10-28 20:34:38,795] INFO : Step 13200/50000 rank6 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0928 | Regression_energy: 0.1004 | Regression_eta: 0.0792 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0688 | LR: 8.54e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.0406s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank6 [2026-10-28 20:38:50,954] INFO : Step 13300/50000 rank6 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1081 | Regression_energy: 0.1155 | Regression_eta: 0.1090 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1099 | LR: 8.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.4887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank6 [2026-10-28 20:43:04,348] INFO : Step 13400/50000 rank6 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2529 | Regression_cos_phi: 0.1207 | Regression_energy: 0.0945 | Regression_eta: 0.0814 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0983 | LR: 8.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2847s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank6 [2026-10-28 20:47:13,658] INFO : Step 13500/50000 rank6 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0860 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0952 | Regression_eta: 0.0740 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0798 | LR: 8.48e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1207s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank6 [2026-10-28 20:51:26,251] INFO : Step 13600/50000 rank6 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2242 | Regression_cos_phi: 0.1121 | Regression_energy: 0.1003 | Regression_eta: 0.0786 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0822 | LR: 8.45e-04 | DataLoad Time: 0.0140s | Model Forward Time: 3.7991s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank6 [2026-10-28 20:55:41,399] INFO : Step 13700/50000 rank6 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.3003 | Regression_cos_phi: 0.1029 | Regression_energy: 0.0975 | Regression_eta: 0.0930 | Regression_pt: 0.1020 | Regression_sin_phi: 0.1209 | LR: 8.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.8275s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank6 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank6 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2300 | Regression_cos_phi: 0.1160 | Regression_energy: 0.1154 | Regression_eta: 0.0721 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0900 | LR: 8.41e-04 | DataLoad Time: 0.0227s | Model Forward Time: 6.1136s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank6 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank6 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0609 | Regression_energy: 0.1138 | Regression_eta: 0.0599 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0731 | LR: 8.39e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.1586s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank6 [2026-10-28 21:08:24,062] INFO : Step 14000/50000 rank6 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2814 | Regression_cos_phi: 0.1153 | Regression_energy: 0.0929 | Regression_eta: 0.0824 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0929 | LR: 8.37e-04 | DataLoad Time: 0.0137s | Model Forward Time: 5.9221s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank6 [2026-10-28 21:08:24,998] INFO : Running validation on rank6 +rank6 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:09:16,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 21:09:43,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:09:43,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:09:44,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:09:45,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:09:55,949] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:09:56,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:09:56,997] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 21:10:13,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:10:14,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:10:14,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:10:15,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:10:25,909] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:10:27,036] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 21:10:42,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:10:43,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:10:45,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:10:46,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:10:56,138] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank6 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0911 | Regression_eta: 0.0826 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0604 | LR: 8.35e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5876s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank6 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank6 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0920 | Regression_energy: 0.0925 | Regression_eta: 0.0662 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0895 | LR: 8.32e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4861s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank6 [2026-10-28 21:24:03,128] INFO : Step 14300/50000 rank6 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2172 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0952 | Regression_eta: 0.0944 | Regression_pt: 0.1002 | Regression_sin_phi: 0.1238 | LR: 8.30e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0238s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank6 [2026-10-28 21:28:09,726] INFO : Step 14400/50000 rank6 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2847 | Regression_cos_phi: 0.0686 | Regression_energy: 0.1086 | Regression_eta: 0.0695 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0871 | LR: 8.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0248s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank6 [2026-10-28 21:32:11,131] INFO : Step 14500/50000 rank6 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0852 | Regression_eta: 0.0688 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0818 | LR: 8.26e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.3469s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank6 [2026-10-28 21:36:11,052] INFO : Step 14600/50000 rank6 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2738 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0907 | Regression_eta: 0.0685 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0880 | LR: 8.24e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.9812s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank6 [2026-10-28 21:40:16,614] INFO : Step 14700/50000 rank6 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2624 | Regression_cos_phi: 0.1059 | Regression_energy: 0.0812 | Regression_eta: 0.0702 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0746 | LR: 8.21e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1363s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank6 [2026-10-28 21:44:33,909] INFO : Step 14800/50000 rank6 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2433 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0945 | Regression_eta: 0.0735 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0986 | LR: 8.19e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3497s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank6 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank6 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0906 | Regression_eta: 0.0465 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0631 | LR: 8.17e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.3240s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank6 [2026-10-28 21:52:53,292] INFO : Step 15000/50000 rank6 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2630 | Regression_cos_phi: 0.1537 | Regression_energy: 0.1017 | Regression_eta: 0.1105 | Regression_pt: 0.1074 | Regression_sin_phi: 0.1114 | LR: 8.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7245s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank6 [2026-10-28 21:52:54,210] INFO : Running validation on rank6 +rank6 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:53:44,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:53:44,574] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 21:54:10,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:54:11,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:54:12,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:54:13,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:54:23,936] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:54:24,123] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:54:24,597] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 21:54:40,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:54:41,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:54:42,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:54:43,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:54:53,554] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:54:54,914] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 21:55:10,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:55:11,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:55:11,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:55:12,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:55:22,579] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 21:59:40,819] INFO : Step 15100/50000 rank6 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2555 | Regression_cos_phi: 0.0599 | Regression_energy: 0.1016 | Regression_eta: 0.1028 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1030 | LR: 8.12e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.6887s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank6 [2026-10-28 22:03:40,545] INFO : Step 15200/50000 rank6 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0853 | Regression_eta: 0.0646 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0480 | LR: 8.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9627s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank6 [2026-10-28 22:07:35,577] INFO : Step 15300/50000 rank6 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0850 | Regression_eta: 0.0600 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0751 | LR: 8.08e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8192s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank6 [2026-10-28 22:11:47,343] INFO : Step 15400/50000 rank6 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2721 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0820 | Regression_eta: 0.0829 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1177 | LR: 8.05e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1747s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank6 [2026-10-28 22:16:06,325] INFO : Step 15500/50000 rank6 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0931 | Regression_eta: 0.0648 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0757 | LR: 8.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9209s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank6 [2026-10-28 22:20:18,326] INFO : Step 15600/50000 rank6 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0870 | Regression_eta: 0.0536 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0679 | LR: 8.01e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.4197s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank6 [2026-10-28 22:24:30,252] INFO : Step 15700/50000 rank6 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0956 | Classification_binary: 0.2702 | Regression_cos_phi: 0.1251 | Regression_energy: 0.0951 | Regression_eta: 0.0913 | Regression_pt: 0.0982 | Regression_sin_phi: 0.1085 | LR: 7.98e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.9296s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank6 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank6 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2337 | Regression_cos_phi: 0.1034 | Regression_energy: 0.1037 | Regression_eta: 0.0743 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0978 | LR: 7.96e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3020s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank6 [2026-10-28 22:32:57,712] INFO : Step 15900/50000 rank6 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0739 | Regression_eta: 0.0599 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0666 | LR: 7.94e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.3914s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank6 [2026-10-28 22:37:10,741] INFO : Step 16000/50000 rank6 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2512 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0858 | Regression_eta: 0.0787 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0873 | LR: 7.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0614s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank6 [2026-10-28 22:37:11,681] INFO : Running validation on rank6 +rank6 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 22:38:03,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:38:04,031] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 22:38:30,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 22:38:31,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 22:38:32,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 22:38:33,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 22:38:43,478] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:38:43,782] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:38:44,322] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 22:39:00,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 22:39:01,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 22:39:03,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 22:39:04,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 22:39:14,673] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 22:39:15,560] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:39:16,106] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 22:39:31,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 22:39:32,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 22:39:33,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 22:39:33,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 22:39:43,786] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 22:44:03,866] INFO : Step 16100/50000 rank6 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0781 | Regression_eta: 0.0618 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0761 | LR: 7.89e-04 | DataLoad Time: 0.0092s | Model Forward Time: 4.7590s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank6 [2026-10-28 22:48:09,285] INFO : Step 16200/50000 rank6 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0883 | Regression_eta: 0.0699 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0850 | LR: 7.86e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.6654s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank6 [2026-10-28 22:52:16,253] INFO : Step 16300/50000 rank6 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2622 | Regression_cos_phi: 0.1262 | Regression_energy: 0.1005 | Regression_eta: 0.1042 | Regression_pt: 0.1057 | Regression_sin_phi: 0.1235 | LR: 7.84e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1391s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank6 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank6 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2742 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0738 | Regression_eta: 0.0720 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0780 | LR: 7.81e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.4173s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank6 [2026-10-28 23:00:23,723] INFO : Step 16500/50000 rank6 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0929 | Regression_eta: 0.0885 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0905 | LR: 7.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9997s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank6 [2026-10-28 23:04:26,317] INFO : Step 16600/50000 rank6 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0913 | Regression_eta: 0.0522 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0786 | LR: 7.77e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.3626s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank6 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank6 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2298 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0781 | Regression_eta: 0.0715 | Regression_pt: 0.0818 | Regression_sin_phi: 0.1050 | LR: 7.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0573s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank6 [2026-10-28 23:12:28,973] INFO : Step 16800/50000 rank6 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2304 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0692 | Regression_eta: 0.0501 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0672 | LR: 7.72e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.0649s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank6 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank6 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0923 | Regression_eta: 0.0439 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0748 | LR: 7.69e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7645s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank6 [2026-10-28 23:20:36,991] INFO : Step 17000/50000 rank6 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0904 | Classification_binary: 0.2563 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0799 | Regression_eta: 0.0821 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1067 | LR: 7.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9361s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank6 [2026-10-28 23:20:37,935] INFO : Running validation on rank6 +rank6 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 23:21:28,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:21:29,342] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 23:21:56,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 23:21:57,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 23:22:00,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 23:22:02,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 23:22:12,123] INFO : Time taken to make predictions on device 6 is: 0.32 min +rank6 [2026-10-28 23:22:12,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:22:12,959] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 23:22:29,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 23:22:30,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 23:22:31,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 23:22:32,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 23:22:42,051] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 23:22:42,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:22:42,877] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 23:22:58,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 23:22:59,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 23:22:59,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 23:23:00,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 23:23:10,750] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 23:27:38,315] INFO : Step 17100/50000 rank6 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0758 | Regression_eta: 0.0705 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0630 | LR: 7.64e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.1671s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank6 [2026-10-28 23:31:38,672] INFO : Step 17200/50000 rank6 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0684 | Regression_eta: 0.0720 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0869 | LR: 7.62e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1961s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank6 [2026-10-28 23:35:49,549] INFO : Step 17300/50000 rank6 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2169 | Regression_cos_phi: 0.1100 | Regression_energy: 0.0833 | Regression_eta: 0.0785 | Regression_pt: 0.0892 | Regression_sin_phi: 0.1181 | LR: 7.59e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.5226s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank6 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank6 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2668 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0945 | Regression_eta: 0.0672 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0757 | LR: 7.57e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.3528s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank6 [2026-10-28 23:43:53,976] INFO : Step 17500/50000 rank6 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0816 | Regression_eta: 0.0589 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0757 | LR: 7.54e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.9633s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank6 [2026-10-28 23:48:08,875] INFO : Step 17600/50000 rank6 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0862 | Regression_eta: 0.0763 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0539 | LR: 7.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5565s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank6 [2026-10-28 23:52:17,700] INFO : Step 17700/50000 rank6 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1958 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0836 | Regression_eta: 0.0537 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0684 | LR: 7.49e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8658s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank6 [2026-10-28 23:56:40,919] INFO : Step 17800/50000 rank6 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2111 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0811 | Regression_eta: 0.0582 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0582 | LR: 7.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1253s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank6 [2026-10-29 00:00:47,138] INFO : Step 17900/50000 rank6 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0972 | Regression_eta: 0.0626 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0630 | LR: 7.44e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2720s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank6 [2026-10-29 00:04:53,689] INFO : Step 18000/50000 rank6 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2427 | Regression_cos_phi: 0.0557 | Regression_energy: 0.1015 | Regression_eta: 0.0700 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0686 | LR: 7.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 13.5879s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank6 [2026-10-29 00:04:54,634] INFO : Running validation on rank6 +rank6 [2026-10-29 00:04:54,635] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 00:04:54,635] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:05:43,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:05:43,820] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 00:06:10,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:06:11,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:06:12,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:06:14,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:06:24,173] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 00:06:24,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 00:06:41,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:06:42,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:06:43,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:06:44,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:06:54,077] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 00:07:11,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:07:11,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:07:12,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:07:13,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:07:23,209] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 00:11:38,771] INFO : Step 18100/50000 rank6 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1110 | Regression_energy: 0.0646 | Regression_eta: 0.0689 | Regression_pt: 0.0674 | Regression_sin_phi: 0.1225 | LR: 7.39e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.2930s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank6 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank6 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0706 | Regression_eta: 0.0511 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0819 | LR: 7.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1155s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank6 [2026-10-29 00:19:54,086] INFO : Step 18300/50000 rank6 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0958 | Regression_energy: 0.0871 | Regression_eta: 0.0700 | Regression_pt: 0.0922 | Regression_sin_phi: 0.1051 | LR: 7.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8059s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank6 [2026-10-29 00:24:07,389] INFO : Step 18400/50000 rank6 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2644 | Regression_cos_phi: 0.1093 | Regression_energy: 0.0737 | Regression_eta: 0.0840 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0709 | LR: 7.31e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.4811s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank6 [2026-10-29 00:27:44,478] INFO : Step 18500/50000 rank6 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0687 | Regression_eta: 0.0608 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0623 | LR: 7.29e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank6 [2026-10-29 00:31:24,472] INFO : Step 18600/50000 rank6 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0866 | Regression_eta: 0.0699 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0962 | LR: 7.26e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1898s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank6 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank6 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0732 | Regression_eta: 0.0612 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0984 | LR: 7.24e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.9500s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank6 [2026-10-29 00:38:49,579] INFO : Step 18800/50000 rank6 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0932 | Regression_eta: 0.0581 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0772 | LR: 7.21e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4341s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank6 [2026-10-29 00:42:28,238] INFO : Step 18900/50000 rank6 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1783 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0880 | Regression_eta: 0.0911 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0664 | LR: 7.18e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0108s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank6 [2026-10-29 00:46:10,109] INFO : Step 19000/50000 rank6 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0823 | Regression_eta: 0.0678 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0865 | LR: 7.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1516s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank6 [2026-10-29 00:46:11,054] INFO : Running validation on rank6 +rank6 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:46:58,593] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:46:59,174] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 00:47:25,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:47:26,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:47:27,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:47:28,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:47:38,275] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:47:38,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:47:38,900] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 00:47:54,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:47:55,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:47:56,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:47:57,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:48:07,855] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:48:09,173] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 00:48:24,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:48:25,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:48:25,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:48:26,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:48:36,582] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank6 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0841 | Regression_eta: 0.0553 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0524 | LR: 7.13e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5898s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank6 [2026-10-29 00:56:31,489] INFO : Step 19200/50000 rank6 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0652 | Regression_eta: 0.0539 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0908 | LR: 7.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7702s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank6 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank6 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0765 | Regression_eta: 0.0809 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0955 | LR: 7.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6446s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank6 [2026-10-29 01:03:57,848] INFO : Step 19400/50000 rank6 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2216 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0906 | Regression_eta: 0.0807 | Regression_pt: 0.0939 | Regression_sin_phi: 0.1170 | LR: 7.05e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1741s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank6 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank6 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0811 | Regression_eta: 0.0670 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0691 | LR: 7.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0506s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank6 [2026-10-29 01:11:25,842] INFO : Step 19600/50000 rank6 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1656 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0772 | Regression_eta: 0.0669 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0644 | LR: 7.00e-04 | DataLoad Time: 0.0225s | Model Forward Time: 3.9370s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank6 [2026-10-29 01:15:28,511] INFO : Step 19700/50000 rank6 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0850 | Regression_eta: 0.0641 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0739 | LR: 6.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.3550s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank6 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank6 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2627 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0934 | Regression_eta: 0.0878 | Regression_pt: 0.0984 | Regression_sin_phi: 0.1028 | LR: 6.94e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3383s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank6 [2026-10-29 01:23:26,313] INFO : Step 19900/50000 rank6 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0835 | Regression_eta: 0.0529 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0637 | LR: 6.92e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7476s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank6 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank6 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2325 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0814 | Regression_eta: 0.0825 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0859 | LR: 6.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8758s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank6 [2026-10-29 01:27:16,848] INFO : Running validation on rank6 +rank6 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 01:28:06,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:28:07,178] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 01:28:33,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 01:28:35,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 01:28:36,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 01:28:37,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 01:28:47,265] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 01:28:48,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:28:48,586] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 01:29:04,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 01:29:05,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 01:29:06,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 01:29:07,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 01:29:17,510] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:29:18,019] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:29:18,549] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 01:29:34,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 01:29:34,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 01:29:35,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 01:29:36,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 01:29:46,225] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 01:33:54,475] INFO : Step 20100/50000 rank6 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2554 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0792 | Regression_eta: 0.0830 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1006 | LR: 6.86e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.4715s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank6 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank6 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0725 | Regression_eta: 0.0904 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0985 | LR: 6.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9220s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank6 [2026-10-29 01:41:55,631] INFO : Step 20300/50000 rank6 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0661 | Regression_eta: 0.0395 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0561 | LR: 6.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6955s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank6 [2026-10-29 01:45:46,289] INFO : Step 20400/50000 rank6 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2322 | Regression_cos_phi: 0.1178 | Regression_energy: 0.0743 | Regression_eta: 0.0549 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0797 | LR: 6.78e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1282s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank6 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank6 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2637 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0884 | Regression_eta: 0.0727 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0836 | LR: 6.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3410s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank6 [2026-10-29 01:53:32,129] INFO : Step 20600/50000 rank6 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0770 | Regression_eta: 0.0773 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0727 | LR: 6.73e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.0716s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank6 [2026-10-29 01:57:29,941] INFO : Step 20700/50000 rank6 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2169 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0783 | Regression_eta: 0.0556 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1124 | LR: 6.70e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.6604s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank6 [2026-10-29 02:01:35,426] INFO : Step 20800/50000 rank6 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2934 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0800 | Regression_eta: 0.0984 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0889 | LR: 6.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8002s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank6 [2026-10-29 02:05:49,595] INFO : Step 20900/50000 rank6 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0821 | Regression_eta: 0.0576 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0577 | LR: 6.65e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.1299s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank6 [2026-10-29 02:09:53,643] INFO : Step 21000/50000 rank6 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0987 | Regression_energy: 0.0842 | Regression_eta: 0.0756 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0762 | LR: 6.62e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.8419s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank6 [2026-10-29 02:09:54,611] INFO : Running validation on rank6 +rank6 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:10:45,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:10:45,900] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 02:11:12,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:11:13,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:11:14,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:11:15,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:11:25,714] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:11:26,307] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 02:11:43,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:11:44,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:11:45,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:11:46,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:11:56,939] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 02:11:57,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:11:57,965] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 02:12:13,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:12:13,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:12:14,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:12:15,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:12:25,307] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 02:16:43,091] INFO : Step 21100/50000 rank6 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2504 | Regression_cos_phi: 0.1240 | Regression_energy: 0.0919 | Regression_eta: 0.0940 | Regression_pt: 0.0988 | Regression_sin_phi: 0.1093 | LR: 6.59e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5469s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank6 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank6 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank6 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0778 | Regression_energy: 0.1111 | Regression_eta: 0.0622 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0748 | LR: 6.56e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1636s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank6 [2026-10-29 02:24:40,425] INFO : Step 21300/50000 rank6 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2234 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0842 | Regression_eta: 0.0606 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0792 | LR: 6.54e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1527s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank6 [2026-10-29 02:28:35,146] INFO : Step 21400/50000 rank6 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2712 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0905 | Regression_eta: 0.0705 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0740 | LR: 6.51e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.4152s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank6 [2026-10-29 02:32:39,449] INFO : Step 21500/50000 rank6 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0900 | Regression_energy: 0.0837 | Regression_eta: 0.0830 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0857 | LR: 6.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7855s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank6 [2026-10-29 02:36:35,118] INFO : Step 21600/50000 rank6 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0877 | Regression_eta: 0.0707 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0717 | LR: 6.45e-04 | DataLoad Time: 0.0233s | Model Forward Time: 3.8419s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank6 [2026-10-29 02:40:30,746] INFO : Step 21700/50000 rank6 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1067 | Regression_energy: 0.0788 | Regression_eta: 0.0844 | Regression_pt: 0.0817 | Regression_sin_phi: 0.1115 | LR: 6.43e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.5956s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank6 [2026-10-29 02:44:20,671] INFO : Step 21800/50000 rank6 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2525 | Regression_cos_phi: 0.1069 | Regression_energy: 0.0967 | Regression_eta: 0.0960 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1143 | LR: 6.40e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1210s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank6 [2026-10-29 02:48:12,872] INFO : Step 21900/50000 rank6 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0943 | Regression_energy: 0.0855 | Regression_eta: 0.0718 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0835 | LR: 6.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2318s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank6 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank6 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2307 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0950 | Regression_eta: 0.0694 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0790 | LR: 6.34e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1567s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank6 [2026-10-29 02:52:09,891] INFO : Running validation on rank6 +rank6 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:52:57,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:52:58,578] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 02:53:26,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:53:28,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:53:29,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:53:30,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:53:40,048] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 02:53:56,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:53:57,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:53:58,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:53:59,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:54:09,480] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:54:10,678] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 02:54:26,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:54:26,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:54:27,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:54:28,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:54:38,181] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 02:58:46,448] INFO : Step 22100/50000 rank6 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2724 | Regression_cos_phi: 0.1610 | Regression_energy: 0.1158 | Regression_eta: 0.1198 | Regression_pt: 0.1185 | Regression_sin_phi: 0.1247 | LR: 6.32e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4335s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank6 [2026-10-29 03:02:36,686] INFO : Step 22200/50000 rank6 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0800 | Regression_energy: 0.1081 | Regression_eta: 0.0714 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0890 | LR: 6.29e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6552s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank6 [2026-10-29 03:06:28,833] INFO : Step 22300/50000 rank6 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0880 | Regression_eta: 0.0524 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0757 | LR: 6.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8520s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank6 [2026-10-29 03:10:23,119] INFO : Step 22400/50000 rank6 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2706 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0977 | Regression_eta: 0.0802 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0887 | LR: 6.23e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5951s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank6 [2026-10-29 03:14:16,069] INFO : Step 22500/50000 rank6 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2304 | Regression_cos_phi: 0.1010 | Regression_energy: 0.1117 | Regression_eta: 0.0956 | Regression_pt: 0.1159 | Regression_sin_phi: 0.1235 | LR: 6.20e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8993s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank6 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank6 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0952 | Regression_eta: 0.0681 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0958 | LR: 6.18e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5809s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank6 [2026-10-29 03:22:04,588] INFO : Step 22700/50000 rank6 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0850 | Regression_energy: 0.1045 | Regression_eta: 0.0739 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0645 | LR: 6.15e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3275s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank6 [2026-10-29 03:26:06,878] INFO : Step 22800/50000 rank6 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2484 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0941 | Regression_eta: 0.0916 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0988 | LR: 6.12e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2775s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank6 [2026-10-29 03:30:17,845] INFO : Step 22900/50000 rank6 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2028 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0982 | Regression_eta: 0.0737 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0834 | LR: 6.09e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.6086s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank6 [2026-10-29 03:34:03,730] INFO : Step 23000/50000 rank6 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0924 | Classification_binary: 0.2569 | Regression_cos_phi: 0.0987 | Regression_energy: 0.1087 | Regression_eta: 0.0842 | Regression_pt: 0.1148 | Regression_sin_phi: 0.1228 | LR: 6.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6391s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank6 [2026-10-29 03:34:04,743] INFO : Running validation on rank6 +rank6 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 03:34:52,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:34:53,013] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 03:35:18,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 03:35:19,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 03:35:20,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 03:35:21,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 03:35:31,988] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:35:32,955] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 03:35:48,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 03:35:49,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 03:35:50,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 03:35:51,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 03:36:01,494] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:36:02,378] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:36:02,911] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 03:36:18,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 03:36:19,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 03:36:20,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 03:36:20,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 03:36:30,783] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 03:40:33,161] INFO : Step 23100/50000 rank6 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2363 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0912 | Regression_eta: 0.0668 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0843 | LR: 6.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4135s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank6 [2026-10-29 03:44:14,416] INFO : Step 23200/50000 rank6 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2065 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1042 | Regression_eta: 0.0885 | Regression_pt: 0.1077 | Regression_sin_phi: 0.1133 | LR: 6.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4824s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank6 [2026-10-29 03:47:55,308] INFO : Step 23300/50000 rank6 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.3051 | Regression_cos_phi: 0.1174 | Regression_energy: 0.1042 | Regression_eta: 0.1133 | Regression_pt: 0.1075 | Regression_sin_phi: 0.1090 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9631s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank6 [2026-10-29 03:51:33,797] INFO : Step 23400/50000 rank6 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2567 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0905 | Regression_eta: 0.0976 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0883 | LR: 5.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1970s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank6 [2026-10-29 03:55:22,653] INFO : Step 23500/50000 rank6 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0919 | Regression_eta: 0.0474 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0748 | LR: 5.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0136s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank6 [2026-10-29 03:59:15,167] INFO : Step 23600/50000 rank6 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0767 | Regression_energy: 0.1085 | Regression_eta: 0.0765 | Regression_pt: 0.1148 | Regression_sin_phi: 0.1187 | LR: 5.90e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.9376s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank6 [2026-10-29 04:03:05,414] INFO : Step 23700/50000 rank6 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2529 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0998 | Regression_eta: 0.0723 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0872 | LR: 5.87e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3863s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank6 [2026-10-29 04:06:50,323] INFO : Step 23800/50000 rank6 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2414 | Regression_cos_phi: 0.0920 | Regression_energy: 0.1077 | Regression_eta: 0.0794 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0799 | LR: 5.84e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.0034s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank6 [2026-10-29 04:11:04,730] INFO : Step 23900/50000 rank6 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1873 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0849 | Regression_eta: 0.0579 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0916 | LR: 5.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3367s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank6 [2026-10-29 04:14:57,720] INFO : Step 24000/50000 rank6 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2330 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0998 | Regression_eta: 0.0579 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0802 | LR: 5.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1431s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank6 [2026-10-29 04:14:58,739] INFO : Running validation on rank6 +rank6 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:15:47,909] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:15:48,539] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 04:16:15,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:16:16,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:16:17,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:16:18,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:16:28,654] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:16:29,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 04:16:45,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:16:46,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:16:47,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:16:48,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:16:58,133] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:16:59,583] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 04:17:14,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:17:15,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:17:16,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:17:16,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:17:26,790] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 04:21:44,037] INFO : Step 24100/50000 rank6 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2500 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0938 | Regression_eta: 0.0741 | Regression_pt: 0.0971 | Regression_sin_phi: 0.1027 | LR: 5.75e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.7543s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank6 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank6 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2164 | Regression_cos_phi: 0.0878 | Regression_energy: 0.1067 | Regression_eta: 0.0726 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0887 | LR: 5.73e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2939s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank6 [2026-10-29 04:29:22,270] INFO : Step 24300/50000 rank6 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0875 | Regression_energy: 0.1220 | Regression_eta: 0.0722 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0933 | LR: 5.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0420s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank6 [2026-10-29 04:33:07,495] INFO : Step 24400/50000 rank6 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2729 | Regression_cos_phi: 0.0934 | Regression_energy: 0.1242 | Regression_eta: 0.0954 | Regression_pt: 0.1290 | Regression_sin_phi: 0.1276 | LR: 5.67e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2383s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank6 [2026-10-29 04:36:57,190] INFO : Step 24500/50000 rank6 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2468 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1613 | Regression_eta: 0.1039 | Regression_pt: 0.1661 | Regression_sin_phi: 0.1178 | LR: 5.64e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3248s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank6 [2026-10-29 04:40:40,561] INFO : Step 24600/50000 rank6 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0705 | Regression_energy: 0.1350 | Regression_eta: 0.0584 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0869 | LR: 5.61e-04 | DataLoad Time: 0.0262s | Model Forward Time: 3.9885s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank6 [2026-10-29 04:44:29,988] INFO : Step 24700/50000 rank6 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0915 | Classification_binary: 0.2805 | Regression_cos_phi: 0.1483 | Regression_energy: 0.1275 | Regression_eta: 0.1095 | Regression_pt: 0.1359 | Regression_sin_phi: 0.1574 | LR: 5.58e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5091s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank6 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank6 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0995 | Regression_eta: 0.0968 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0941 | LR: 5.56e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8640s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank6 [2026-10-29 04:52:00,596] INFO : Step 24900/50000 rank6 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0739 | Regression_energy: 0.1058 | Regression_eta: 0.0581 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0688 | LR: 5.53e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9773s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank6 [2026-10-29 04:55:44,333] INFO : Step 25000/50000 rank6 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2410 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1392 | Regression_eta: 0.0699 | Regression_pt: 0.1476 | Regression_sin_phi: 0.1004 | LR: 5.50e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2486s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank6 [2026-10-29 04:55:45,304] INFO : Running validation on rank6 +rank6 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:55:45,305] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:56:33,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:56:34,567] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 04:57:00,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:57:01,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:57:02,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:57:03,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:57:13,124] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:57:13,409] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:57:13,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 04:57:29,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:57:30,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:57:32,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:57:33,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:57:43,110] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:57:43,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:57:44,417] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 04:58:00,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:58:00,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:58:01,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:58:02,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:58:12,179] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 05:02:15,911] INFO : Step 25100/50000 rank6 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2549 | Regression_cos_phi: 0.1233 | Regression_energy: 0.0971 | Regression_eta: 0.0784 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0760 | LR: 5.47e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.7060s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank6 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank6 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.2457 | Regression_cos_phi: 0.1474 | Regression_energy: 0.1280 | Regression_eta: 0.0967 | Regression_pt: 0.1312 | Regression_sin_phi: 0.1436 | LR: 5.44e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2810s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank6 [2026-10-29 05:09:52,833] INFO : Step 25300/50000 rank6 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2237 | Regression_cos_phi: 0.1029 | Regression_energy: 0.1142 | Regression_eta: 0.0820 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0986 | LR: 5.42e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.8551s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank6 [2026-10-29 05:13:35,656] INFO : Step 25400/50000 rank6 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2672 | Regression_cos_phi: 0.1107 | Regression_energy: 0.1205 | Regression_eta: 0.0928 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1261 | LR: 5.39e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0672s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank6 [2026-10-29 05:17:27,534] INFO : Step 25500/50000 rank6 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.2127 | Regression_cos_phi: 0.0945 | Regression_energy: 0.1075 | Regression_eta: 0.0789 | Regression_pt: 0.1136 | Regression_sin_phi: 0.1279 | LR: 5.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0753s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank6 [2026-10-29 05:21:16,671] INFO : Step 25600/50000 rank6 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0856 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0940 | Regression_energy: 0.1291 | Regression_eta: 0.0646 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0848 | LR: 5.33e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8745s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank6 [2026-10-29 05:25:00,203] INFO : Step 25700/50000 rank6 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2845 | Regression_cos_phi: 0.0965 | Regression_energy: 0.1143 | Regression_eta: 0.0751 | Regression_pt: 0.1223 | Regression_sin_phi: 0.1347 | LR: 5.30e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8871s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank6 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank6 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2925 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1197 | Regression_eta: 0.1020 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0985 | LR: 5.27e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5171s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank6 [2026-10-29 05:32:33,909] INFO : Step 25900/50000 rank6 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2223 | Regression_cos_phi: 0.1157 | Regression_energy: 0.1239 | Regression_eta: 0.0847 | Regression_pt: 0.1293 | Regression_sin_phi: 0.1058 | LR: 5.25e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8718s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank6 [2026-10-29 05:36:28,635] INFO : Step 26000/50000 rank6 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2175 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1152 | Regression_eta: 0.0879 | Regression_pt: 0.1201 | Regression_sin_phi: 0.1024 | LR: 5.22e-04 | DataLoad Time: 0.0176s | Model Forward Time: 9.2330s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank6 [2026-10-29 05:36:29,553] INFO : Running validation on rank6 +rank6 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 05:37:19,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:37:20,020] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 05:37:45,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 05:37:46,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 05:37:47,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 05:37:48,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 05:37:58,158] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:37:58,606] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 05:38:15,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 05:38:16,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 05:38:17,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 05:38:18,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 05:38:28,341] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:38:28,895] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 05:38:44,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 05:38:44,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 05:38:45,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 05:38:46,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 05:38:56,292] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 05:43:04,442] INFO : Step 26100/50000 rank6 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 0.2980 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1172 | Regression_eta: 0.0917 | Regression_pt: 0.1232 | Regression_sin_phi: 0.1028 | LR: 5.19e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8580s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank6 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank6 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0601 | Regression_energy: 0.1068 | Regression_eta: 0.0601 | Regression_pt: 0.1106 | Regression_sin_phi: 0.0801 | LR: 5.16e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2706s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank6 [2026-10-29 05:50:52,485] INFO : Step 26300/50000 rank6 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1099 | Classification_binary: 0.2263 | Regression_cos_phi: 0.1199 | Regression_energy: 0.1190 | Regression_eta: 0.1071 | Regression_pt: 0.1239 | Regression_sin_phi: 0.1700 | LR: 5.13e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3842s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank6 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank6 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.2852 | Regression_cos_phi: 0.1856 | Regression_energy: 0.1310 | Regression_eta: 0.1251 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1316 | LR: 5.10e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3829s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank6 [2026-10-29 05:58:42,120] INFO : Step 26500/50000 rank6 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2907 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1083 | Regression_eta: 0.1050 | Regression_pt: 0.1101 | Regression_sin_phi: 0.1158 | LR: 5.08e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4045s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank6 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank6 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2054 | Regression_cos_phi: 0.1065 | Regression_energy: 0.1263 | Regression_eta: 0.0809 | Regression_pt: 0.1291 | Regression_sin_phi: 0.1035 | LR: 5.05e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7662s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank6 [2026-10-29 06:06:39,600] INFO : Step 26700/50000 rank6 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0899 | Regression_energy: 0.1142 | Regression_eta: 0.0699 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0977 | LR: 5.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.6697s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank6 [2026-10-29 06:10:41,253] INFO : Step 26800/50000 rank6 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2859 | Regression_cos_phi: 0.0909 | Regression_energy: 0.1306 | Regression_eta: 0.0755 | Regression_pt: 0.1358 | Regression_sin_phi: 0.1019 | LR: 4.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2394s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank6 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank6 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2141 | Regression_cos_phi: 0.0875 | Regression_energy: 0.1093 | Regression_eta: 0.0667 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0749 | LR: 4.96e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.8902s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank6 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank6 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0935 | Regression_eta: 0.0571 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0622 | LR: 4.94e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.9886s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank6 [2026-10-29 06:18:43,933] INFO : Running validation on rank6 +rank6 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 06:19:32,614] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:19:33,151] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 06:20:00,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 06:20:01,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 06:20:02,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 06:20:03,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 06:20:13,847] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:20:14,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:20:14,780] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 06:20:30,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 06:20:32,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 06:20:33,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 06:20:34,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 06:20:44,906] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:20:46,295] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 06:21:02,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 06:21:02,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 06:21:03,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 06:21:04,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 06:21:14,397] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 06:25:28,750] INFO : Step 27100/50000 rank6 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2803 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1226 | Regression_eta: 0.0779 | Regression_pt: 0.1277 | Regression_sin_phi: 0.1077 | LR: 4.91e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7015s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank6 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank6 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0925 | Regression_energy: 0.1159 | Regression_eta: 0.0616 | Regression_pt: 0.1166 | Regression_sin_phi: 0.1024 | LR: 4.88e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.5752s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank6 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank6 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1060 | Regression_energy: 0.1106 | Regression_eta: 0.0673 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0893 | LR: 4.85e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.9953s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank6 [2026-10-29 06:37:36,741] INFO : Step 27400/50000 rank6 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2710 | Regression_cos_phi: 0.1068 | Regression_energy: 0.1125 | Regression_eta: 0.0832 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1079 | LR: 4.82e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5089s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank6 [2026-10-29 06:41:28,997] INFO : Step 27500/50000 rank6 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2774 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1107 | Regression_eta: 0.0849 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1067 | LR: 4.80e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9222s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank6 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank6 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0870 | Regression_eta: 0.0616 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0684 | LR: 4.77e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8537s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank6 [2026-10-29 06:49:16,010] INFO : Step 27700/50000 rank6 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0967 | Regression_energy: 0.1256 | Regression_eta: 0.0872 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0986 | LR: 4.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0200s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank6 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank6 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2481 | Regression_cos_phi: 0.1195 | Regression_energy: 0.1085 | Regression_eta: 0.0684 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0990 | LR: 4.71e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.6678s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank6 [2026-10-29 06:57:17,959] INFO : Step 27900/50000 rank6 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0569 | Regression_energy: 0.1153 | Regression_eta: 0.0876 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0722 | LR: 4.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5362s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank6 [2026-10-29 07:01:10,312] INFO : Step 28000/50000 rank6 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0902 | Regression_energy: 0.1107 | Regression_eta: 0.0772 | Regression_pt: 0.1143 | Regression_sin_phi: 0.1202 | LR: 4.66e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3506s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank6 [2026-10-29 07:01:11,153] INFO : Running validation on rank6 +rank6 [2026-10-29 07:01:11,154] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:01:11,154] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:02:01,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:02:02,348] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 07:02:30,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:02:31,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:02:32,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:02:33,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:02:43,082] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:02:44,046] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 07:03:00,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:03:01,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:03:02,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:03:03,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:03:13,587] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:03:14,473] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 07:03:29,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:03:30,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:03:31,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:03:31,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:03:41,683] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 07:08:04,578] INFO : Step 28100/50000 rank6 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2669 | Regression_cos_phi: 0.1450 | Regression_energy: 0.1275 | Regression_eta: 0.0908 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0979 | LR: 4.63e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1484s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank6 [2026-10-29 07:11:56,590] INFO : Step 28200/50000 rank6 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0836 | Regression_energy: 0.1208 | Regression_eta: 0.0735 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0949 | LR: 4.60e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank6 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank6 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1968 | Regression_cos_phi: 0.1066 | Regression_energy: 0.1173 | Regression_eta: 0.0848 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0814 | LR: 4.57e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8320s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank6 [2026-10-29 07:20:03,893] INFO : Step 28400/50000 rank6 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2149 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1099 | Regression_eta: 0.0791 | Regression_pt: 0.1136 | Regression_sin_phi: 0.1224 | LR: 4.55e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.0875s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank6 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank6 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2873 | Regression_cos_phi: 0.1433 | Regression_energy: 0.1465 | Regression_eta: 0.0987 | Regression_pt: 0.1514 | Regression_sin_phi: 0.1050 | LR: 4.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1338s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank6 [2026-10-29 07:27:52,535] INFO : Step 28600/50000 rank6 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0865 | Regression_energy: 0.1013 | Regression_eta: 0.0692 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0545 | LR: 4.49e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2898s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank6 [2026-10-29 07:31:49,173] INFO : Step 28700/50000 rank6 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0914 | Regression_energy: 0.1086 | Regression_eta: 0.0559 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1016 | LR: 4.46e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0944s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank6 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank6 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2726 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1130 | Regression_eta: 0.0785 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0894 | LR: 4.44e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4535s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank6 [2026-10-29 07:39:38,710] INFO : Step 28900/50000 rank6 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2175 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0929 | Regression_eta: 0.0804 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0928 | LR: 4.41e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3187s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank6 [2026-10-29 07:43:43,865] INFO : Step 29000/50000 rank6 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0881 | Regression_energy: 0.1198 | Regression_eta: 0.0706 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0710 | LR: 4.38e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9074s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank6 [2026-10-29 07:43:44,819] INFO : Running validation on rank6 +rank6 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:44:33,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:44:33,713] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 07:44:59,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:45:00,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:45:01,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:45:02,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:45:12,565] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:45:12,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:45:13,372] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 07:45:28,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:45:29,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:45:30,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:45:32,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:45:42,010] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:45:43,719] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 07:45:59,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:45:59,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:46:00,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:46:01,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:46:11,301] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 07:50:42,137] INFO : Step 29100/50000 rank6 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1017 | Regression_energy: 0.0966 | Regression_eta: 0.0539 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0735 | LR: 4.35e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.6989s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank6 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank6 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0949 | Regression_energy: 0.1018 | Regression_eta: 0.0625 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0927 | LR: 4.33e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.2684s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank6 [2026-10-29 07:58:22,884] INFO : Step 29300/50000 rank6 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0724 | Regression_energy: 0.1011 | Regression_eta: 0.0471 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0673 | LR: 4.30e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0002s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank6 [2026-10-29 08:02:11,406] INFO : Step 29400/50000 rank6 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0937 | Regression_energy: 0.1000 | Regression_eta: 0.0805 | Regression_pt: 0.1040 | Regression_sin_phi: 0.1085 | LR: 4.27e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5171s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank6 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank6 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2671 | Regression_cos_phi: 0.0887 | Regression_energy: 0.1048 | Regression_eta: 0.0564 | Regression_pt: 0.1098 | Regression_sin_phi: 0.1016 | LR: 4.24e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4984s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank6 [2026-10-29 08:09:49,626] INFO : Step 29600/50000 rank6 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0771 | Regression_energy: 0.1082 | Regression_eta: 0.0592 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0603 | LR: 4.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7102s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank6 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank6 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0681 | Regression_energy: 0.1080 | Regression_eta: 0.0683 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0782 | LR: 4.19e-04 | DataLoad Time: 0.0126s | Model Forward Time: 4.1782s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank6 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank6 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2486 | Regression_cos_phi: 0.0797 | Regression_energy: 0.1001 | Regression_eta: 0.0641 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0931 | LR: 4.16e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3852s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank6 [2026-10-29 08:21:10,555] INFO : Step 29900/50000 rank6 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0941 | Regression_eta: 0.0421 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0757 | LR: 4.14e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9962s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank6 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank6 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0828 | Regression_eta: 0.0664 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0834 | LR: 4.11e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7031s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank6 [2026-10-29 08:24:57,340] INFO : Running validation on rank6 +rank6 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 08:25:45,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:25:45,651] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 08:26:11,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 08:26:12,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 08:26:13,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 08:26:14,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 08:26:24,460] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 08:26:24,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:26:25,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 08:26:41,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 08:26:42,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 08:26:43,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 08:26:44,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 08:26:54,485] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:26:55,053] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 08:27:10,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 08:27:11,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 08:27:12,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 08:27:12,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 08:27:22,891] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 08:31:31,210] INFO : Step 30100/50000 rank6 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2380 | Regression_cos_phi: 0.1184 | Regression_energy: 0.0968 | Regression_eta: 0.1065 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1048 | LR: 4.08e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8518s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank6 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank6 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2381 | Regression_cos_phi: 0.0748 | Regression_energy: 0.1057 | Regression_eta: 0.0683 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0826 | LR: 4.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5333s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank6 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank6 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0796 | Regression_eta: 0.0583 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0794 | LR: 4.03e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.1795s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank6 [2026-10-29 08:43:19,167] INFO : Step 30400/50000 rank6 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0854 | Regression_eta: 0.0540 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0737 | LR: 4.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5065s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank6 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank6 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2537 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0920 | Regression_eta: 0.1004 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0755 | LR: 3.98e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7624s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank6 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank6 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0701 | Regression_eta: 0.0558 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0680 | LR: 3.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5468s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank6 [2026-10-29 08:54:35,590] INFO : Step 30700/50000 rank6 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0782 | Regression_eta: 0.0738 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0911 | LR: 3.92e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2953s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank6 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank6 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2468 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0890 | Regression_eta: 0.0560 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0928 | LR: 3.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2069s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank6 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank6 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0849 | Regression_eta: 0.0606 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0669 | LR: 3.87e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9284s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank6 [2026-10-29 09:05:52,365] INFO : Step 31000/50000 rank6 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0799 | Regression_energy: 0.1047 | Regression_eta: 0.0533 | Regression_pt: 0.1098 | Regression_sin_phi: 0.1183 | LR: 3.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1901s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank6 [2026-10-29 09:05:53,358] INFO : Running validation on rank6 +rank6 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:06:41,697] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:06:42,226] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 09:07:08,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:07:09,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:07:10,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:07:11,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:07:21,572] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:07:22,582] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 09:07:38,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:07:39,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:07:40,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:07:41,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:07:51,122] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:07:52,508] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 09:08:08,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:08:09,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:08:09,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:08:10,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:08:20,494] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank6 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1794 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0912 | Regression_eta: 0.0703 | Regression_pt: 0.0974 | Regression_sin_phi: 0.1083 | LR: 3.82e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.4601s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank6 [2026-10-29 09:16:29,206] INFO : Step 31200/50000 rank6 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2623 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0811 | Regression_eta: 0.0700 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0744 | LR: 3.79e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5348s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank6 [2026-10-29 09:20:24,830] INFO : Step 31300/50000 rank6 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0775 | Regression_eta: 0.0620 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1110 | LR: 3.76e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.4677s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank6 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank6 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0841 | Regression_eta: 0.0809 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0929 | LR: 3.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8360s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank6 [2026-10-29 09:28:17,179] INFO : Step 31500/50000 rank6 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2431 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0912 | Regression_eta: 0.0703 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0965 | LR: 3.71e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.3615s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank6 [2026-10-29 09:32:16,798] INFO : Step 31600/50000 rank6 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0976 | Regression_energy: 0.0743 | Regression_eta: 0.0438 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0573 | LR: 3.69e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8634s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank6 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank6 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0876 | Regression_eta: 0.0516 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0704 | LR: 3.66e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.1347s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank6 [2026-10-29 09:40:22,622] INFO : Step 31800/50000 rank6 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0732 | Regression_eta: 0.0454 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0579 | LR: 3.64e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3546s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank6 [2026-10-29 09:48:15,345] INFO : Step 31900/50000 rank6 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0926 | Regression_eta: 0.0707 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0688 | LR: 3.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8693s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank6 [2026-10-29 09:52:25,844] INFO : Step 32000/50000 rank6 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0935 | Regression_eta: 0.0527 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0800 | LR: 3.58e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1225s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank6 [2026-10-29 09:52:26,792] INFO : Running validation on rank6 +rank6 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:53:17,424] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 09:53:46,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:53:47,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:53:48,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:53:49,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:53:59,722] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:54:01,008] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 09:54:16,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:54:17,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:54:18,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:54:19,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:54:29,700] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:54:30,788] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 09:54:46,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:54:47,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:54:48,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:54:48,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:54:58,711] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank6 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0991 | Regression_eta: 0.0666 | Regression_pt: 0.1064 | Regression_sin_phi: 0.1088 | LR: 3.56e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9468s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank6 [2026-10-29 10:03:26,458] INFO : Step 32200/50000 rank6 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2289 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0912 | Regression_eta: 0.0691 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0879 | LR: 3.53e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5232s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank6 [2026-10-29 10:07:22,791] INFO : Step 32300/50000 rank6 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0679 | Regression_eta: 0.0616 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0633 | LR: 3.51e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5865s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank6 [2026-10-29 10:11:42,986] INFO : Step 32400/50000 rank6 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0814 | Regression_eta: 0.0656 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0764 | LR: 3.48e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.0263s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank6 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank6 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2450 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0935 | Regression_eta: 0.0559 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0815 | LR: 3.46e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.5246s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank6 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank6 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0839 | Regression_eta: 0.0625 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0868 | LR: 3.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4751s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank6 [2026-10-29 10:24:11,750] INFO : Step 32700/50000 rank6 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0808 | Regression_eta: 0.0442 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0630 | LR: 3.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0507s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank6 [2026-10-29 10:28:05,809] INFO : Step 32800/50000 rank6 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2326 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0769 | Regression_eta: 0.0500 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0856 | LR: 3.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5519s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank6 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank6 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0968 | Regression_eta: 0.0675 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0771 | LR: 3.36e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.8076s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank6 [2026-10-29 10:36:14,934] INFO : Step 33000/50000 rank6 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0809 | Regression_eta: 0.0491 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0614 | LR: 3.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7663s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank6 [2026-10-29 10:36:15,962] INFO : Running validation on rank6 +rank6 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 10:37:03,787] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:37:04,370] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 10:37:32,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 10:37:34,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 10:37:35,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 10:37:37,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 10:37:47,608] INFO : Time taken to make predictions on device 6 is: 0.30 min +rank6 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:37:48,814] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 10:38:08,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 10:38:09,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 10:38:10,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 10:38:11,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 10:38:21,187] INFO : Time taken to make predictions on device 6 is: 0.30 min +rank6 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:38:21,704] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 10:38:37,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 10:38:38,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 10:38:38,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 10:38:39,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 10:38:49,598] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 10:43:12,058] INFO : Step 33100/50000 rank6 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0333 | Regression_energy: 0.0718 | Regression_eta: 0.0505 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0470 | LR: 3.31e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9482s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank6 [2026-10-29 10:47:22,844] INFO : Step 33200/50000 rank6 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0700 | Regression_eta: 0.0653 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0609 | LR: 3.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6977s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank6 [2026-10-29 10:51:33,084] INFO : Step 33300/50000 rank6 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0738 | Regression_eta: 0.0737 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0871 | LR: 3.26e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4514s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank6 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank6 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0863 | Regression_eta: 0.0632 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0681 | LR: 3.23e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank6 [2026-10-29 10:59:48,622] INFO : Step 33500/50000 rank6 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0821 | Regression_eta: 0.0618 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0760 | LR: 3.21e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.9160s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank6 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank6 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0735 | Regression_eta: 0.0504 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0570 | LR: 3.19e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.2008s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank6 [2026-10-29 11:09:08,677] INFO : Step 33700/50000 rank6 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0714 | Regression_eta: 0.0854 | Regression_pt: 0.0723 | Regression_sin_phi: 0.1114 | LR: 3.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3152s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank6 [2026-10-29 11:13:02,294] INFO : Step 33800/50000 rank6 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0833 | Regression_eta: 0.0491 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0810 | LR: 3.14e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3677s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank6 [2026-10-29 11:16:52,541] INFO : Step 33900/50000 rank6 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2388 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0815 | Regression_eta: 0.0640 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0699 | LR: 3.11e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.8124s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank6 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank6 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0883 | Regression_eta: 0.0574 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0858 | LR: 3.09e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9502s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank6 [2026-10-29 11:20:48,852] INFO : Running validation on rank6 +rank6 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 11:21:42,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:21:42,909] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 11:22:09,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 11:22:10,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 11:22:11,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 11:22:12,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 11:22:22,282] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:22:22,979] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 11:22:39,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 11:22:40,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 11:22:41,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 11:22:42,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 11:22:52,086] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:22:53,477] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 11:23:09,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 11:23:09,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 11:23:10,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 11:23:11,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 11:23:21,207] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 11:27:42,518] INFO : Step 34100/50000 rank6 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0838 | Regression_eta: 0.0654 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0642 | LR: 3.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.5186s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank6 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank6 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2573 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0821 | Regression_eta: 0.0725 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0709 | LR: 3.04e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.7874s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank6 [2026-10-29 11:35:56,847] INFO : Step 34300/50000 rank6 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0975 | Regression_eta: 0.0451 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0664 | LR: 3.02e-04 | DataLoad Time: 0.0173s | Model Forward Time: 7.2753s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank6 [2026-10-29 11:40:08,368] INFO : Step 34400/50000 rank6 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1624 | Regression_cos_phi: 0.1366 | Regression_energy: 0.0875 | Regression_eta: 0.0735 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0776 | LR: 2.99e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.9074s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank6 [2026-10-29 11:44:04,820] INFO : Step 34500/50000 rank6 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0816 | Regression_eta: 0.0472 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0787 | LR: 2.97e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5072s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank6 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank6 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0882 | Regression_eta: 0.0780 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0711 | LR: 2.95e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6068s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank6 [2026-10-29 11:52:00,087] INFO : Step 34700/50000 rank6 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0815 | Regression_eta: 0.0578 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0767 | LR: 2.92e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.5678s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank6 [2026-10-29 11:55:53,802] INFO : Step 34800/50000 rank6 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0941 | Regression_eta: 0.0581 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0651 | LR: 2.90e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8593s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank6 [2026-10-29 11:59:58,708] INFO : Step 34900/50000 rank6 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0697 | Regression_eta: 0.0499 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0704 | LR: 2.88e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank6 [2026-10-29 12:04:06,317] INFO : Step 35000/50000 rank6 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0853 | Regression_eta: 0.0700 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0536 | LR: 2.85e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4572s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank6 [2026-10-29 12:04:07,348] INFO : Running validation on rank6 +rank6 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 12:05:04,842] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:05:05,387] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 12:05:31,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 12:05:32,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 12:05:33,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 12:05:34,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 12:05:44,934] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:05:45,909] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 12:06:02,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 12:06:02,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 12:06:03,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 12:06:04,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 12:06:14,902] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 12:06:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:06:16,080] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 12:06:32,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 12:06:32,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 12:06:33,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 12:06:34,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 12:06:44,086] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank6 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0786 | Regression_eta: 0.0579 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0510 | LR: 2.83e-04 | DataLoad Time: 0.0396s | Model Forward Time: 5.6228s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank6 [2026-10-29 12:15:06,128] INFO : Step 35200/50000 rank6 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2307 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0869 | Regression_eta: 0.0626 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0744 | LR: 2.81e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0570s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank6 [2026-10-29 12:18:57,447] INFO : Step 35300/50000 rank6 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0789 | Regression_eta: 0.0418 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0469 | LR: 2.79e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8166s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank6 [2026-10-29 12:23:00,201] INFO : Step 35400/50000 rank6 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0970 | Regression_eta: 0.0608 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0942 | LR: 2.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3944s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank6 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank6 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0834 | Regression_eta: 0.0437 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0543 | LR: 2.74e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2695s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank6 [2026-10-29 12:31:17,344] INFO : Step 35600/50000 rank6 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0800 | Regression_eta: 0.0623 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0660 | LR: 2.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1600s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank6 [2026-10-29 12:35:31,256] INFO : Step 35700/50000 rank6 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0895 | Regression_eta: 0.0368 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0561 | LR: 2.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2774s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank6 [2026-10-29 14:49:26,285] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-29 14:49:26,293] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-29 14:49:53,254] INFO : Instantiating model +rank6 [2026-10-29 14:49:53,254] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-29 14:49:53,254] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-29 14:49:53,273] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-29 14:49:53,273] INFO : nn0 parameters: 171776 +rank6 [2026-10-29 14:49:53,273] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank6 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 1 +rank6 [2026-10-29 14:49:53,279] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,284] INFO : Initializing backbone layer 2 +rank6 [2026-10-29 14:49:53,284] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,290] INFO : Initializing backbone layer 3 +rank6 [2026-10-29 14:49:53,290] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,295] INFO : Initializing backbone layer 4 +rank6 [2026-10-29 14:49:53,295] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,300] INFO : Initializing backbone layer 5 +rank6 [2026-10-29 14:49:53,300] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:49:53,305] INFO : Backbone initialization took 0.03s +rank6 [2026-10-29 14:49:53,306] INFO : backbone parameters: 2374656 +rank6 [2026-10-29 14:49:53,306] INFO : Initializing output DNNs +rank6 [2026-10-29 14:49:53,358] INFO : Output DNNs initialization took 0.05s +rank6 [2026-10-29 14:49:53,358] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-29 14:49:53,358] INFO : jagged_attention enabled=True +rank6 [2026-10-29 14:49:53,358] INFO : flash_attn_varlen enabled=True +rank6 [2026-10-29 14:49:53,359] INFO : task_queries enabled=False +rank6 [2026-10-29 14:49:53,359] INFO : classification_norm parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : regression_norm parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : classification_query parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : regression_query parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : classification_readout parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : regression_readout parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : output_mode=set +rank6 [2026-10-29 14:49:53,359] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-29 14:49:53,359] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-29 14:49:53,359] INFO : nn_pid parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : nn_pt parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : nn_eta parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : nn_energy parameters: 0 +rank6 [2026-10-29 14:49:53,360] INFO : Total MLPF parameters: 7260244 +rank6 [2026-10-29 14:49:53,360] INFO : MLPF __init__ done +rank6 [2026-10-29 14:49:53,361] INFO : Instantiated model +rank6 [2026-10-29 14:49:53,361] INFO : Moving model to device rank=6 +rank6 [2026-10-29 14:49:53,375] INFO : Moved model to device rank=6 +rank6 [2026-10-29 14:49:53,376] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-29 14:49:53,695] INFO : Restoring training from step 35001 +rank6 [2026-10-29 14:49:53,701] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank6 [2026-10-29 14:49:53,701] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-29 14:49:53,930] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-29 14:49:54,047] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank6 [2026-10-29 14:49:54,047] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-29 14:49:54,048] INFO : Trainable parameters: 7260244 +rank6 [2026-10-29 14:49:54,049] INFO : Non-trainable parameters: 0 +rank6 [2026-10-29 14:49:54,049] INFO : Total parameters: 7260244 +rank6 [2026-10-29 14:49:54,051] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-29 14:49:54,052] INFO : Getting dataloaders +rank6 [2026-10-29 14:49:54,052] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:01,422] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:02,342] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:03,542] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-29 14:50:03,543] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-29 14:50:03,543] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:03,830] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:04,079] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:04,390] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-29 14:50:04,390] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-29 14:50:18,187] INFO : Got dataloaders +rank6 [2026-10-29 14:50:18,187] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank6 [2026-10-29 14:50:18,187] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank6 [2026-10-29 14:50:18,187] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank6 [2026-10-29 14:50:18,187] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank6 [2026-10-29 14:50:19,671] INFO : loader split=train rank=6 len=21108 +rank6 [2026-10-29 14:50:19,672] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-29 14:50:19,672] INFO : Starting training from step 35001 to 50000 on rank 6 of 8 +rank6 [2026-10-29 14:50:19,672] INFO : Creating train_iterator +rank6 [2026-10-29 14:50:19,672] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank6 [2026-10-29 14:50:34,064] INFO : Created train_iterator +rank6 [2026-10-29 14:54:16,364] INFO : Step 35100/50000 rank6 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0776 | Regression_eta: 0.0526 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0753 | LR: 2.83e-04 | DataLoad Time: 0.0167s | Model Forward Time: 2.8680s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank6 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank6 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0778 | Regression_eta: 0.0717 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0862 | LR: 2.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5088s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank6 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank6 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0369 | Regression_energy: 0.0993 | Regression_eta: 0.0378 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0366 | LR: 2.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5916s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank6 [2026-10-29 15:05:10,668] INFO : Step 35400/50000 rank6 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0857 | Regression_eta: 0.0663 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0575 | LR: 2.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.4255s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank6 [2026-10-29 15:08:49,315] INFO : Step 35500/50000 rank6 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0764 | Regression_eta: 0.0467 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0475 | LR: 2.74e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.4550s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank6 [2026-10-29 15:12:27,420] INFO : Step 35600/50000 rank6 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0800 | Regression_eta: 0.0663 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0698 | LR: 2.72e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6893s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank6 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank6 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0924 | Regression_eta: 0.0426 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0616 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.3514s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank6 [2026-10-29 15:19:47,618] INFO : Step 35800/50000 rank6 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2037 | Regression_cos_phi: 0.1155 | Regression_energy: 0.0783 | Regression_eta: 0.0545 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0730 | LR: 2.68e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.4896s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank6 [2026-10-29 15:23:30,654] INFO : Step 35900/50000 rank6 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0757 | Regression_eta: 0.0521 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0904 | LR: 2.65e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8625s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank6 [2026-10-29 15:27:19,182] INFO : Step 36000/50000 rank6 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0321 | Regression_energy: 0.0903 | Regression_eta: 0.0343 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0826 | LR: 2.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8289s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank6 [2026-10-29 15:27:20,114] INFO : Running validation on rank6 +rank6 [2026-10-29 15:27:20,114] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 15:28:08,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:28:08,877] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 15:28:36,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 15:28:37,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 15:28:38,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 15:28:39,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 15:28:49,644] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 15:28:49,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:28:50,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 15:29:05,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 15:29:06,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 15:29:07,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 15:29:08,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 15:29:18,689] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:29:20,198] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 15:29:35,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 15:29:36,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 15:29:37,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 15:29:37,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 15:29:47,996] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank6 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0748 | Regression_eta: 0.0719 | Regression_pt: 0.0798 | Regression_sin_phi: 0.1429 | LR: 2.61e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.9084s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank6 [2026-10-29 15:37:36,021] INFO : Step 36200/50000 rank6 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0832 | Regression_eta: 0.0592 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0877 | LR: 2.59e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5676s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank6 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank6 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0652 | Regression_eta: 0.0524 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0626 | LR: 2.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5107s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank6 [2026-10-29 15:45:12,807] INFO : Step 36400/50000 rank6 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0619 | Regression_eta: 0.0419 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0493 | LR: 2.55e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.2448s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank6 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank6 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0755 | Regression_eta: 0.0599 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0590 | LR: 2.52e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.0158s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank6 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank6 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0757 | Regression_eta: 0.0561 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0693 | LR: 2.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7035s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank6 [2026-10-29 15:56:51,300] INFO : Step 36700/50000 rank6 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0684 | Regression_eta: 0.0309 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0533 | LR: 2.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7863s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank6 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank6 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0906 | Regression_eta: 0.0807 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0935 | LR: 2.46e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4171s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank6 [2026-10-29 16:04:39,809] INFO : Step 36900/50000 rank6 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0868 | Regression_eta: 0.0685 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0918 | LR: 2.44e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7925s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank6 [2026-10-29 16:08:30,144] INFO : Step 37000/50000 rank6 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0876 | Regression_eta: 0.0528 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0772 | LR: 2.42e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9055s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank6 [2026-10-29 16:08:31,022] INFO : Running validation on rank6 +rank6 [2026-10-29 16:08:31,022] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:09:20,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:09:20,760] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 16:09:46,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:09:47,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:09:48,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:09:49,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:09:59,419] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 16:10:16,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:10:17,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:10:18,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:10:19,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:10:29,127] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:10:29,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:10:30,425] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 16:10:46,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:10:46,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:10:47,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:10:48,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:10:58,285] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 16:15:06,402] INFO : Step 37100/50000 rank6 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0735 | Regression_eta: 0.0543 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0723 | LR: 2.40e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8495s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank6 [2026-10-29 16:18:46,282] INFO : Step 37200/50000 rank6 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0690 | Regression_eta: 0.0532 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0764 | LR: 2.38e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2300s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank6 [2026-10-29 16:22:16,729] INFO : Step 37300/50000 rank6 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0852 | Regression_eta: 0.0511 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0690 | LR: 2.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5812s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank6 [2026-10-29 16:26:01,823] INFO : Step 37400/50000 rank6 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0881 | Regression_eta: 0.0439 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0503 | LR: 2.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9410s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank6 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank6 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0705 | Regression_eta: 0.0526 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0661 | LR: 2.32e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3475s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank6 [2026-10-29 16:34:14,939] INFO : Step 37600/50000 rank6 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1791 | Regression_cos_phi: 0.1185 | Regression_energy: 0.0860 | Regression_eta: 0.0490 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0658 | LR: 2.30e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1113s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank6 [2026-10-29 16:38:00,808] INFO : Step 37700/50000 rank6 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0824 | Regression_eta: 0.0654 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0626 | LR: 2.28e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1661s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank6 [2026-10-29 16:41:51,410] INFO : Step 37800/50000 rank6 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0845 | Regression_eta: 0.0540 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0729 | LR: 2.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.2929s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank6 [2026-10-29 16:45:39,811] INFO : Step 37900/50000 rank6 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2431 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0745 | Regression_eta: 0.0622 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0733 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8080s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank6 [2026-10-29 16:49:32,192] INFO : Step 38000/50000 rank6 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0908 | Regression_eta: 0.0620 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0954 | LR: 2.22e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8887s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank6 [2026-10-29 16:49:33,136] INFO : Running validation on rank6 +rank6 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:49:33,136] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:50:21,275] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 16:50:47,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:50:49,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:50:50,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:50:53,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:51:03,500] INFO : Time taken to make predictions on device 6 is: 0.29 min +rank6 [2026-10-29 16:51:06,265] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:51:06,744] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 16:51:22,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:51:23,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:51:24,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:51:25,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:51:35,508] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:51:36,911] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 16:51:52,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:51:53,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:51:54,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:51:55,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:52:05,035] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank6 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0842 | Regression_eta: 0.0478 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0901 | LR: 2.20e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3704s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank6 [2026-10-29 17:00:01,013] INFO : Step 38200/50000 rank6 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2156 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0961 | Regression_eta: 0.0632 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0935 | LR: 2.18e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8297s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank6 [2026-10-29 17:03:44,996] INFO : Step 38300/50000 rank6 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0825 | Regression_eta: 0.0440 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0606 | LR: 2.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2343s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank6 [2026-10-29 17:07:28,968] INFO : Step 38400/50000 rank6 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0870 | Regression_eta: 0.0471 | Regression_pt: 0.0879 | Regression_sin_phi: 0.1080 | LR: 2.14e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.5973s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank6 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank6 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0671 | Regression_eta: 0.0581 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0555 | LR: 2.12e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.5366s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank6 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank6 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0793 | Regression_eta: 0.0513 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0964 | LR: 2.11e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4120s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank6 [2026-10-29 17:18:55,030] INFO : Step 38700/50000 rank6 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0714 | Regression_eta: 0.0429 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0546 | LR: 2.09e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.8735s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank6 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank6 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0796 | Regression_eta: 0.0451 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0647 | LR: 2.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5798s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank6 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank6 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0797 | Regression_eta: 0.0563 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0633 | LR: 2.05e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6664s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank6 [2026-10-29 17:30:46,441] INFO : Step 39000/50000 rank6 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0780 | Regression_eta: 0.0504 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0709 | LR: 2.03e-04 | DataLoad Time: 0.0225s | Model Forward Time: 3.6465s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank6 [2026-10-29 17:30:47,444] INFO : Running validation on rank6 +rank6 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 17:31:35,653] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:31:36,208] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 17:32:01,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 17:32:02,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 17:32:03,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 17:32:04,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 17:32:14,941] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:32:15,856] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 17:32:31,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 17:32:32,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 17:32:33,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 17:32:34,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 17:32:44,204] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:32:45,862] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 17:33:01,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 17:33:01,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 17:33:02,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 17:33:03,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 17:33:13,404] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank6 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0819 | Regression_eta: 0.0598 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0589 | LR: 2.01e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6377s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank6 [2026-10-29 17:41:18,056] INFO : Step 39200/50000 rank6 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2107 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0958 | Regression_eta: 0.0776 | Regression_pt: 0.1017 | Regression_sin_phi: 0.1231 | LR: 2.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7455s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank6 [2026-10-29 17:45:05,747] INFO : Step 39300/50000 rank6 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0813 | Regression_eta: 0.0569 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0745 | LR: 1.98e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.9366s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank6 [2026-10-29 17:49:01,211] INFO : Step 39400/50000 rank6 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0711 | Regression_eta: 0.0346 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0425 | LR: 1.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0455s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank6 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank6 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0813 | Regression_eta: 0.0548 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0565 | LR: 1.94e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2580s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank6 [2026-10-29 17:56:59,043] INFO : Step 39600/50000 rank6 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0807 | Regression_eta: 0.0809 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0772 | LR: 1.93e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1943s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank6 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank6 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0797 | Regression_eta: 0.0519 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0770 | LR: 1.91e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2351s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank6 [2026-10-29 18:04:32,041] INFO : Step 39800/50000 rank6 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0965 | Regression_eta: 0.0822 | Regression_pt: 0.0992 | Regression_sin_phi: 0.1015 | LR: 1.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8990s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank6 [2026-10-29 18:08:19,425] INFO : Step 39900/50000 rank6 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2326 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0814 | Regression_eta: 0.0680 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0856 | LR: 1.88e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1831s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank6 [2026-10-29 18:12:05,433] INFO : Step 40000/50000 rank6 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0849 | Regression_eta: 0.0652 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0752 | LR: 1.86e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9395s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank6 [2026-10-29 18:12:06,327] INFO : Running validation on rank6 +rank6 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:12:59,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:12:59,647] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 18:13:23,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:13:24,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:13:25,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:13:26,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:13:36,521] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:13:37,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:13:37,720] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 18:13:53,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:13:54,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:13:55,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:13:56,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:14:06,619] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:14:07,478] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 18:14:22,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:14:23,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:14:24,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:14:25,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:14:35,059] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 18:18:48,522] INFO : Step 40100/50000 rank6 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0913 | Regression_eta: 0.0460 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0561 | LR: 1.84e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1848s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank6 [2026-10-29 18:22:51,410] INFO : Step 40200/50000 rank6 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0934 | Regression_eta: 0.0478 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0750 | LR: 1.83e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.9263s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank6 [2026-10-29 18:26:53,014] INFO : Step 40300/50000 rank6 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2459 | Regression_cos_phi: 0.1093 | Regression_energy: 0.0849 | Regression_eta: 0.0664 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0893 | LR: 1.81e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4861s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank6 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank6 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1957 | Regression_cos_phi: 0.1111 | Regression_energy: 0.1001 | Regression_eta: 0.0649 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0780 | LR: 1.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0122s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank6 [2026-10-29 18:34:31,872] INFO : Step 40500/50000 rank6 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0963 | Regression_eta: 0.0402 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0517 | LR: 1.78e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.4815s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank6 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank6 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0871 | Regression_eta: 0.0484 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0720 | LR: 1.76e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1592s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank6 [2026-10-29 18:42:09,669] INFO : Step 40700/50000 rank6 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0887 | Regression_eta: 0.0447 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0465 | LR: 1.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5721s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank6 [2026-10-29 18:45:59,793] INFO : Step 40800/50000 rank6 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0933 | Regression_eta: 0.0462 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0621 | LR: 1.73e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.0687s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank6 [2026-10-29 18:49:49,168] INFO : Step 40900/50000 rank6 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0959 | Regression_eta: 0.0491 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0617 | LR: 1.72e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6411s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank6 [2026-10-29 18:53:41,415] INFO : Step 41000/50000 rank6 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0974 | Regression_eta: 0.0610 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0635 | LR: 1.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4682s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank6 [2026-10-29 18:53:42,383] INFO : Running validation on rank6 +rank6 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:54:33,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:54:33,853] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 18:54:59,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:55:00,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:55:01,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:55:02,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:55:12,716] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:55:13,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:55:13,608] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 18:55:29,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:55:30,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:55:31,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:55:32,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:55:42,494] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 18:55:59,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:56:00,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:56:01,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:56:01,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:56:11,696] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 19:00:33,893] INFO : Step 41100/50000 rank6 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0847 | Regression_eta: 0.0540 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0681 | LR: 1.69e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.8485s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank6 [2026-10-29 19:04:28,229] INFO : Step 41200/50000 rank6 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0870 | Regression_eta: 0.0798 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0603 | LR: 1.67e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9458s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank6 [2026-10-29 19:08:18,072] INFO : Step 41300/50000 rank6 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2062 | Regression_cos_phi: 0.0958 | Regression_energy: 0.0932 | Regression_eta: 0.0560 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0678 | LR: 1.66e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1400s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank6 [2026-10-29 19:12:02,347] INFO : Step 41400/50000 rank6 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0886 | Regression_eta: 0.0494 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0556 | LR: 1.64e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9388s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank6 [2026-10-29 19:15:48,707] INFO : Step 41500/50000 rank6 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0654 | Regression_energy: 0.1066 | Regression_eta: 0.0480 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0696 | LR: 1.63e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.3259s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank6 [2026-10-29 19:19:48,124] INFO : Step 41600/50000 rank6 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2140 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0949 | Regression_eta: 0.0557 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0613 | LR: 1.61e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7646s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank6 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank6 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0651 | Regression_energy: 0.1116 | Regression_eta: 0.0643 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0763 | LR: 1.60e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.9469s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank6 [2026-10-29 19:27:38,847] INFO : Step 41800/50000 rank6 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0879 | Regression_eta: 0.0704 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0670 | LR: 1.58e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.3153s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank6 [2026-10-29 19:31:32,457] INFO : Step 41900/50000 rank6 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1791 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1092 | Regression_eta: 0.0687 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0860 | LR: 1.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4523s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank6 [2026-10-29 19:35:32,890] INFO : Step 42000/50000 rank6 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2268 | Regression_cos_phi: 0.0735 | Regression_energy: 0.1120 | Regression_eta: 0.0496 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0931 | LR: 1.56e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9402s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank6 [2026-10-29 19:35:33,871] INFO : Running validation on rank6 +rank6 [2026-10-29 19:35:33,871] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 19:36:22,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:36:23,273] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 19:36:50,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 19:36:51,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 19:36:52,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 19:36:53,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 19:37:03,092] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:37:04,355] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 19:37:20,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 19:37:21,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 19:37:22,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 19:37:23,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 19:37:33,259] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 19:37:33,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:37:34,031] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 19:37:49,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 19:37:49,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 19:37:50,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 19:37:51,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 19:38:01,389] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 19:42:30,056] INFO : Step 42100/50000 rank6 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0823 | Regression_eta: 0.0406 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0537 | LR: 1.54e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0599s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank6 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank6 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2330 | Regression_cos_phi: 0.1088 | Regression_energy: 0.1045 | Regression_eta: 0.0679 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0845 | LR: 1.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8373s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank6 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank6 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 19:50:51,512] INFO : Step 42300/50000 rank6 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0895 | Regression_eta: 0.0540 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0514 | LR: 1.52e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2016s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank6 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank6 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0791 | Regression_eta: 0.0519 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0572 | LR: 1.50e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.6748s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank6 [2026-10-29 19:58:48,120] INFO : Step 42500/50000 rank6 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0936 | Regression_eta: 0.0556 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0663 | LR: 1.49e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1944s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank6 [2026-10-29 20:02:59,631] INFO : Step 42600/50000 rank6 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.2054 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0914 | Regression_eta: 0.0462 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0548 | LR: 1.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 8.8077s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank6 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank6 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0745 | Regression_eta: 0.0626 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0560 | LR: 1.47e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7997s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank6 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank6 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0793 | Regression_eta: 0.0430 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0742 | LR: 1.45e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5869s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank6 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank6 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2305 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0846 | Regression_eta: 0.0526 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0535 | LR: 1.44e-04 | DataLoad Time: 0.0106s | Model Forward Time: 4.4208s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank6 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank6 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0680 | Regression_eta: 0.0648 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0516 | LR: 1.43e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9439s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank6 [2026-10-29 20:18:48,627] INFO : Running validation on rank6 +rank6 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 20:18:48,628] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 20:19:38,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:19:39,385] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 20:20:09,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 20:20:10,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 20:20:11,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 20:20:12,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 20:20:22,820] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:20:23,703] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 20:20:40,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 20:20:41,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 20:20:42,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 20:20:43,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 20:20:53,011] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:20:53,998] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 20:21:09,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 20:21:09,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 20:21:10,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 20:21:11,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 20:21:21,241] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank6 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0861 | Regression_eta: 0.0675 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0753 | LR: 1.42e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.2849s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank6 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank6 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0854 | Regression_eta: 0.0693 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0596 | LR: 1.40e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank6 [2026-10-29 20:33:23,251] INFO : Step 43300/50000 rank6 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0786 | Regression_eta: 0.0611 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0712 | LR: 1.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3857s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank6 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank6 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0842 | Regression_eta: 0.0439 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0652 | LR: 1.38e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5481s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank6 [2026-10-29 20:40:59,928] INFO : Step 43500/50000 rank6 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0954 | Regression_eta: 0.0540 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0530 | LR: 1.37e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1842s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank6 [2026-10-29 20:44:45,320] INFO : Step 43600/50000 rank6 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0900 | Regression_eta: 0.0504 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0703 | LR: 1.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0812s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank6 [2026-10-29 20:48:25,993] INFO : Step 43700/50000 rank6 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0735 | Regression_eta: 0.0442 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0529 | LR: 1.35e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.4606s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank6 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank6 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0771 | Regression_eta: 0.0418 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0562 | LR: 1.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.5350s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank6 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank6 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0778 | Regression_eta: 0.0819 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0851 | LR: 1.33e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3479s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank6 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank6 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0839 | Regression_eta: 0.0362 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0575 | LR: 1.32e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.1939s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank6 [2026-10-29 21:00:02,713] INFO : Running validation on rank6 +rank6 [2026-10-29 21:00:02,714] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:00:02,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:00:57,489] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:00:58,021] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 21:01:24,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:01:25,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:01:26,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:01:27,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:01:37,663] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:01:38,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:01:38,954] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 21:01:54,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:01:55,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:01:56,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:01:57,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:02:07,400] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 21:02:08,376] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:02:08,958] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 21:02:24,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:02:25,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:02:26,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:02:26,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:02:36,704] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 21:06:42,683] INFO : Step 44100/50000 rank6 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0817 | Regression_eta: 0.0471 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0742 | LR: 1.31e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1700s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank6 [2026-10-29 21:10:40,741] INFO : Step 44200/50000 rank6 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0707 | Regression_eta: 0.0408 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0630 | LR: 1.30e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9768s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank6 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank6 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0858 | Regression_eta: 0.0636 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0606 | LR: 1.29e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1183s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank6 [2026-10-29 21:18:20,342] INFO : Step 44400/50000 rank6 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0809 | Regression_eta: 0.0425 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0561 | LR: 1.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7685s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank6 [2026-10-29 21:22:16,363] INFO : Step 44500/50000 rank6 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0777 | Regression_eta: 0.0721 | Regression_pt: 0.0801 | Regression_sin_phi: 0.1374 | LR: 1.27e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.8091s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank6 [2026-10-29 21:26:10,594] INFO : Step 44600/50000 rank6 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2275 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0911 | Regression_eta: 0.0751 | Regression_pt: 0.0944 | Regression_sin_phi: 0.1019 | LR: 1.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1480s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank6 [2026-10-29 21:30:16,074] INFO : Step 44700/50000 rank6 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0888 | Regression_eta: 0.0361 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0434 | LR: 1.25e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6974s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank6 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank6 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0744 | Regression_eta: 0.0518 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0655 | LR: 1.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4090s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank6 [2026-10-29 21:38:11,006] INFO : Step 44900/50000 rank6 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0867 | Regression_eta: 0.0506 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0702 | LR: 1.23e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9543s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank6 [2026-10-29 21:42:13,638] INFO : Step 45000/50000 rank6 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0710 | Regression_energy: 0.1064 | Regression_eta: 0.0666 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0749 | LR: 1.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0107s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank6 [2026-10-29 21:42:14,560] INFO : Running validation on rank6 +rank6 [2026-10-29 21:42:14,560] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:42:14,560] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:43:06,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:43:07,157] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 21:43:32,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:43:33,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:43:34,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:43:36,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:43:46,128] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 21:43:47,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:43:47,810] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 21:44:03,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:44:04,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:44:05,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:44:06,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:44:16,573] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:44:17,713] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 21:44:33,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:44:33,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:44:34,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:44:35,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:44:45,125] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 21:48:55,616] INFO : Step 45100/50000 rank6 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0815 | Regression_eta: 0.0614 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0597 | LR: 1.21e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4107s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank6 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank6 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2075 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0943 | Regression_eta: 0.0554 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0758 | LR: 1.20e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1528s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank6 [2026-10-29 21:56:35,620] INFO : Step 45300/50000 rank6 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0948 | Regression_eta: 0.0643 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0915 | LR: 1.19e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.0128s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank6 [2026-10-29 22:00:32,927] INFO : Step 45400/50000 rank6 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1424 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0854 | Regression_eta: 0.0366 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0583 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9033s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank6 [2026-10-29 22:04:23,231] INFO : Step 45500/50000 rank6 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0886 | Regression_eta: 0.0526 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0804 | LR: 1.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6355s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank6 [2026-10-29 22:08:21,194] INFO : Step 45600/50000 rank6 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0694 | Regression_eta: 0.0372 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0557 | LR: 1.17e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0164s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank6 [2026-10-29 22:12:11,370] INFO : Step 45700/50000 rank6 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0743 | Regression_eta: 0.0543 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0812 | LR: 1.16e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.0845s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank6 [2026-10-29 22:16:08,602] INFO : Step 45800/50000 rank6 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0276 | Classification_binary: 0.1290 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0732 | Regression_eta: 0.0677 | Regression_pt: 0.0771 | Regression_sin_phi: 0.1218 | LR: 1.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6668s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank6 [2026-10-29 22:20:18,709] INFO : Step 45900/50000 rank6 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1895 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0851 | Regression_eta: 0.0634 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0659 | LR: 1.15e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9611s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank6 [2026-10-29 22:24:24,449] INFO : Step 46000/50000 rank6 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0809 | Regression_eta: 0.0425 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0584 | LR: 1.14e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2986s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank6 [2026-10-29 22:24:25,403] INFO : Running validation on rank6 +rank6 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 22:25:16,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:25:17,231] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 22:25:46,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 22:25:47,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 22:25:48,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 22:25:49,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 22:25:59,478] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 22:25:59,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:26:00,492] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 22:26:16,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 22:26:16,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 22:26:18,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 22:26:20,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 22:26:30,132] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:26:31,921] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 22:26:47,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 22:26:48,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 22:26:48,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 22:26:49,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 22:26:59,627] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 22:31:16,603] INFO : Step 46100/50000 rank6 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0840 | Regression_eta: 0.0333 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0531 | LR: 1.13e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0436s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank6 [2026-10-29 22:35:12,809] INFO : Step 46200/50000 rank6 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0867 | Regression_eta: 0.0443 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0731 | LR: 1.13e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.9309s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank6 [2026-10-29 22:39:25,150] INFO : Step 46300/50000 rank6 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0847 | Regression_eta: 0.0430 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0801 | LR: 1.12e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9849s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank6 [2026-10-29 22:43:23,784] INFO : Step 46400/50000 rank6 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0837 | Regression_eta: 0.0506 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0815 | LR: 1.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6661s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank6 [2026-10-29 22:47:17,351] INFO : Step 46500/50000 rank6 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0980 | Regression_eta: 0.0460 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0545 | LR: 1.11e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5735s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank6 [2026-10-29 22:51:20,582] INFO : Step 46600/50000 rank6 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0805 | Regression_eta: 0.0544 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0600 | LR: 1.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.3959s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank6 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank6 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0618 | Regression_eta: 0.0517 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0629 | LR: 1.10e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0454s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank6 [2026-10-29 22:59:26,808] INFO : Step 46800/50000 rank6 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0765 | Regression_eta: 0.0661 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0841 | LR: 1.09e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.9131s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank6 [2026-10-29 23:04:15,567] INFO : Step 46900/50000 rank6 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2182 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0864 | Regression_eta: 0.0644 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0685 | LR: 1.09e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.0390s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank6 [2026-10-29 23:08:15,438] INFO : Step 47000/50000 rank6 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0640 | Regression_eta: 0.0638 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0848 | LR: 1.08e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2704s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank6 [2026-10-29 23:08:16,507] INFO : Running validation on rank6 +rank6 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 23:09:07,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:09:08,184] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 23:09:36,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 23:09:37,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 23:09:40,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 23:09:42,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 23:09:52,228] INFO : Time taken to make predictions on device 6 is: 0.31 min +rank6 [2026-10-29 23:09:55,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:09:55,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 23:10:11,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 23:10:12,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 23:10:13,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 23:10:14,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 23:10:24,512] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:10:25,936] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 23:10:41,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 23:10:41,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 23:10:42,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 23:10:43,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 23:10:53,248] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank6 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0860 | Regression_eta: 0.0649 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0692 | LR: 1.07e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7044s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank6 [2026-10-29 23:19:19,773] INFO : Step 47200/50000 rank6 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0657 | Regression_eta: 0.0759 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0999 | LR: 1.07e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.6780s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank6 [2026-10-29 23:23:31,491] INFO : Step 47300/50000 rank6 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2332 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0901 | Regression_eta: 0.0497 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0899 | LR: 1.06e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7139s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank6 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank6 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0785 | Regression_eta: 0.0478 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0721 | LR: 1.06e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0192s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank6 [2026-10-29 23:31:38,582] INFO : Step 47500/50000 rank6 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0763 | Regression_eta: 0.0557 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0438 | LR: 1.06e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8361s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank6 [2026-10-29 23:35:34,354] INFO : Step 47600/50000 rank6 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0784 | Regression_eta: 0.0321 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0445 | LR: 1.05e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5004s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank6 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank6 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2026 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0941 | Regression_eta: 0.0625 | Regression_pt: 0.1005 | Regression_sin_phi: 0.1115 | LR: 1.05e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8402s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank6 [2026-10-29 23:43:53,969] INFO : Step 47800/50000 rank6 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0791 | Regression_eta: 0.0640 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0744 | LR: 1.04e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.5865s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank6 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank6 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0804 | Regression_eta: 0.0459 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0739 | LR: 1.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1956s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank6 [2026-10-29 23:51:38,726] INFO : Step 48000/50000 rank6 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0862 | Regression_eta: 0.0449 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0799 | LR: 1.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2952s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank6 [2026-10-29 23:51:39,741] INFO : Running validation on rank6 +rank6 [2026-10-29 23:51:39,741] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 23:51:39,741] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 23:52:31,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:52:31,694] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 23:52:58,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 23:53:00,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 23:53:01,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 23:53:02,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 23:53:12,864] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 23:53:13,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 23:53:29,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 23:53:30,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 23:53:31,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 23:53:32,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 23:53:42,185] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:53:43,872] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 23:53:59,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 23:53:59,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 23:54:00,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 23:54:01,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 23:54:11,308] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 00:00:53,530] INFO : Step 48100/50000 rank6 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0679 | Regression_energy: 0.1172 | Regression_eta: 0.0624 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0631 | LR: 1.03e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5278s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank6 [2026-10-30 00:04:40,584] INFO : Step 48200/50000 rank6 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0877 | Regression_eta: 0.0456 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0538 | LR: 1.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0282s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank6 [2026-10-30 00:08:31,629] INFO : Step 48300/50000 rank6 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0820 | Regression_eta: 0.0700 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0671 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9452s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank6 [2026-10-30 00:12:19,574] INFO : Step 48400/50000 rank6 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0836 | Regression_eta: 0.0697 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0566 | LR: 1.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3332s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank6 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank6 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0839 | Regression_eta: 0.0423 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0559 | LR: 1.02e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.5870s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank6 [2026-10-30 00:19:50,481] INFO : Step 48600/50000 rank6 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2257 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0820 | Regression_eta: 0.0496 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0677 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9689s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank6 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank6 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2288 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0828 | Regression_eta: 0.0520 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0937 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0932s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank6 [2026-10-30 00:27:30,340] INFO : Step 48800/50000 rank6 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0798 | Regression_eta: 0.0447 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0672 | LR: 1.01e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4245s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank6 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank6 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0846 | Regression_eta: 0.0432 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0756 | LR: 1.01e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6433s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank6 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank6 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0879 | Regression_eta: 0.0533 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0753 | LR: 1.01e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5983s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank6 [2026-10-30 00:35:15,047] INFO : Running validation on rank6 +rank6 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:36:03,499] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:36:04,035] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 00:36:29,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:36:31,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:36:32,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:36:33,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:36:43,461] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-30 00:36:48,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:36:48,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 00:37:04,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:37:05,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:37:06,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:37:07,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:37:17,084] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 00:37:17,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:37:18,485] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 00:37:33,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 00:37:34,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 00:37:34,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 00:37:35,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 00:37:45,596] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 00:41:49,930] INFO : Step 49100/50000 rank6 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0907 | Regression_eta: 0.0524 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0535 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6172s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank6 [2026-10-30 00:45:59,116] INFO : Step 49200/50000 rank6 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0739 | Regression_eta: 0.0493 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0648 | LR: 1.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5919s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank6 [2026-10-30 00:49:54,419] INFO : Step 49300/50000 rank6 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0781 | Regression_eta: 0.0536 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0713 | LR: 1.00e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9846s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank6 [2026-10-30 00:53:44,253] INFO : Step 49400/50000 rank6 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0910 | Regression_eta: 0.0425 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0665 | LR: 1.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9606s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank6 [2026-10-30 00:57:39,407] INFO : Step 49500/50000 rank6 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0742 | Regression_eta: 0.0443 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0622 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0100s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank6 [2026-10-30 01:01:39,140] INFO : Step 49600/50000 rank6 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1970 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0832 | Regression_eta: 0.0575 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0657 | LR: 1.00e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8036s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank6 [2026-10-30 01:05:30,840] INFO : Step 49700/50000 rank6 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0732 | Regression_eta: 0.0376 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0710 | LR: 1.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4568s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank6 [2026-10-30 01:09:21,933] INFO : Step 49800/50000 rank6 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0753 | Regression_eta: 0.0546 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0610 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3599s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank6 [2026-10-30 01:13:13,492] INFO : Step 49900/50000 rank6 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0768 | Regression_eta: 0.0480 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0577 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0833s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank6 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank6 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0782 | Regression_eta: 0.0675 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0740 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8270s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank6 [2026-10-30 01:17:09,619] INFO : Running validation on rank6 +rank6 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 01:18:02,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:18:03,251] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 01:18:29,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 01:18:30,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 01:18:31,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 01:18:32,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 01:18:42,140] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:18:42,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:18:43,093] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 01:18:59,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 01:19:00,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 01:19:01,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 01:19:01,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 01:19:12,003] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:19:13,323] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 01:19:28,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:19:29,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:19:29,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:19:30,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:19:40,571] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 01:19:41,595] INFO : Training completed. Total time on device 6: 629.365min diff --git a/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.7 b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..4e5895fb88272d37b7e37de00f65384896bfebce --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_attention_seed12345_20261028_111202_121851/train.log.7 @@ -0,0 +1,2483 @@ +rank7 [2026-10-28 11:12:32,468] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-28 11:12:32,469] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-28 11:12:57,431] INFO : Instantiating model +rank7 [2026-10-28 11:12:57,432] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-28 11:12:57,432] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-28 11:12:57,453] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-28 11:12:57,453] INFO : nn0 parameters: 171776 +rank7 [2026-10-28 11:12:57,453] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-28 11:12:57,453] INFO : Initializing backbone layer 0 +rank7 [2026-10-28 11:12:57,453] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,457] INFO : Initializing backbone layer 1 +rank7 [2026-10-28 11:12:57,457] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,460] INFO : Initializing backbone layer 2 +rank7 [2026-10-28 11:12:57,460] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,463] INFO : Initializing backbone layer 3 +rank7 [2026-10-28 11:12:57,463] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,466] INFO : Initializing backbone layer 4 +rank7 [2026-10-28 11:12:57,466] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,469] INFO : Initializing backbone layer 5 +rank7 [2026-10-28 11:12:57,469] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:57,473] INFO : Backbone initialization took 0.02s +rank7 [2026-10-28 11:12:57,473] INFO : backbone parameters: 2374656 +rank7 [2026-10-28 11:12:57,473] INFO : Initializing output DNNs +rank7 [2026-10-28 11:12:57,522] INFO : Output DNNs initialization took 0.05s +rank7 [2026-10-28 11:12:57,522] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-28 11:12:57,523] INFO : jagged_attention enabled=True +rank7 [2026-10-28 11:12:57,523] INFO : flash_attn_varlen enabled=True +rank7 [2026-10-28 11:12:57,523] INFO : task_queries enabled=False +rank7 [2026-10-28 11:12:57,523] INFO : classification_norm parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : regression_norm parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : classification_query parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : regression_query parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : classification_readout parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : regression_readout parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : output_mode=set +rank7 [2026-10-28 11:12:57,523] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-28 11:12:57,523] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-28 11:12:57,523] INFO : nn_pid parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : nn_pt parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : nn_eta parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : nn_energy parameters: 0 +rank7 [2026-10-28 11:12:57,524] INFO : Total MLPF parameters: 7260244 +rank7 [2026-10-28 11:12:57,524] INFO : MLPF __init__ done +rank7 [2026-10-28 11:12:57,525] INFO : Instantiated model +rank7 [2026-10-28 11:12:57,525] INFO : Moving model to device rank=7 +rank7 [2026-10-28 11:12:57,559] INFO : Moved model to device rank=7 +rank7 [2026-10-28 11:12:57,560] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-28 11:12:57,561] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-28 11:12:57,695] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-28 11:12:57,813] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank7 [2026-10-28 11:12:57,813] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-28 11:12:57,814] INFO : Trainable parameters: 7260244 +rank7 [2026-10-28 11:12:57,814] INFO : Non-trainable parameters: 0 +rank7 [2026-10-28 11:12:57,814] INFO : Total parameters: 7260244 +rank7 [2026-10-28 11:12:57,817] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-28 11:12:57,817] INFO : Getting dataloaders +rank7 [2026-10-28 11:12:57,817] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:03,711] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:04,765] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,274] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,549] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,812] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-28 11:13:06,812] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-28 11:13:20,908] INFO : Got dataloaders +rank7 [2026-10-28 11:13:20,908] INFO : loader split=train rank=7 len=21108 +rank7 [2026-10-28 11:13:20,908] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-28 11:13:20,908] INFO : Starting training from step 1 to 50000 on rank 7 of 8 +rank7 [2026-10-28 11:13:20,908] INFO : Creating train_iterator +rank7 [2026-10-28 11:13:20,908] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-28 11:13:35,457] INFO : Created train_iterator +rank7 [2026-10-28 11:17:18,354] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5435764396190643, 'Classification': 0.9201868727058172, 'Regression_pt': 1.0861788396537304, 'Regression_eta': 0.015906925462768413, 'Regression_sin_phi': 0.0071213571043335835, 'Regression_cos_phi': 0.00707158214005176, 'Regression_energy': 1.1240814112126827}, weights={'Classification_binary': 1.0, 'Classification': 0.590723991394043, 'Regression_pt': 0.5004484057426453, 'Regression_eta': 34.17231369018555, 'Regression_sin_phi': 76.33045959472656, 'Regression_cos_phi': 76.86772918701172, 'Regression_energy': 0.48357391357421875} +rank7 [2026-10-28 11:17:20,490] INFO : Step 100/50000 rank7 | Train Loss: 5.8747 | Train Loss Unweighted: 3.7807 | Train Loss Components: Auxiliary: 1.0539 | Cardinality: 0.5395 | Classification: 0.8378 | Classification_binary: 0.4936 | HitGrouping: 0.5005 | Regression_cos_phi: 0.0080 | Regression_energy: 1.2459 | Regression_eta: 0.0210 | Regression_pt: 1.1664 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.9075 | Classification_binary: 0.4819 | Regression_cos_phi: 0.0102 | Regression_energy: 1.5076 | Regression_eta: 0.0232 | Regression_pt: 1.3631 | Regression_sin_phi: 0.0080 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 3.1321s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6033 | max_loss: 0.3041 | max_backward: 1.0611 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4639.9 | uss_MB: 3395.7 | child_rss_MB: 5584.3 | allocated_MB: 343.3 | reserved_MB: 22216.0 | inactive_split_MB: 255.4 +rank7 [2026-10-28 11:20:57,690] INFO : Step 200/50000 rank7 | Train Loss: 4.8801 | Train Loss Unweighted: 2.8042 | Train Loss Components: Auxiliary: 0.8885 | Cardinality: 0.5986 | Classification: 0.7630 | Classification_binary: 0.5353 | HitGrouping: 0.4558 | Regression_cos_phi: 0.0053 | Regression_energy: 0.7290 | Regression_eta: 0.0112 | Regression_pt: 0.7548 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.4453 | Classification_binary: 0.5695 | Regression_cos_phi: 0.4528 | Regression_energy: 0.3451 | Regression_eta: 0.3818 | Regression_pt: 0.3693 | Regression_sin_phi: 0.4498 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 3.5598s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0960 | max_forward: 0.7496 | max_loss: 0.1485 | max_backward: 1.4210 | max_optimizer_step: 0.1430 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4643.1 | uss_MB: 3410.6 | child_rss_MB: 5578.8 | allocated_MB: 345.0 | reserved_MB: 22216.0 | inactive_split_MB: 181.6 +rank7 [2026-10-28 11:24:33,676] INFO : Step 300/50000 rank7 | Train Loss: 3.9729 | Train Loss Unweighted: 2.3148 | Train Loss Components: Auxiliary: 0.7260 | Cardinality: 0.3704 | Classification: 0.5541 | Classification_binary: 0.4579 | HitGrouping: 0.4378 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6498 | Regression_eta: 0.0099 | Regression_pt: 0.6342 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3117 | Classification_binary: 0.4352 | Regression_cos_phi: 0.4047 | Regression_energy: 0.3178 | Regression_eta: 0.3420 | Regression_pt: 0.3244 | Regression_sin_phi: 0.2901 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 3.7566s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.6399 | max_loss: 0.2172 | max_backward: 1.3388 | max_optimizer_step: 0.1559 | max_tensorboard_logging: 1.0005 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4655.8 | uss_MB: 3414.3 | child_rss_MB: 5579.0 | allocated_MB: 342.2 | reserved_MB: 24762.0 | inactive_split_MB: 244.5 +rank7 [2026-10-28 11:28:09,553] INFO : Step 400/50000 rank7 | Train Loss: 3.7856 | Train Loss Unweighted: 2.2892 | Train Loss Components: Auxiliary: 0.7106 | Cardinality: 0.3027 | Classification: 0.5044 | Classification_binary: 0.4419 | HitGrouping: 0.4341 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6654 | Regression_eta: 0.0090 | Regression_pt: 0.6601 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3142 | Classification_binary: 0.4407 | Regression_cos_phi: 0.2370 | Regression_energy: 0.3042 | Regression_eta: 0.2785 | Regression_pt: 0.3074 | Regression_sin_phi: 0.3201 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.5819s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0059 | max_forward: 0.5855 | max_loss: 0.1374 | max_backward: 1.1454 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 1.0740 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4653.4 | uss_MB: 3414.6 | child_rss_MB: 5570.3 | allocated_MB: 341.6 | reserved_MB: 24762.0 | inactive_split_MB: 287.8 +rank7 [2026-10-28 11:31:45,938] INFO : Step 500/50000 rank7 | Train Loss: 4.0865 | Train Loss Unweighted: 2.2942 | Train Loss Components: Auxiliary: 0.7417 | Cardinality: 0.4897 | Classification: 0.5265 | Classification_binary: 0.5170 | HitGrouping: 0.4394 | Regression_cos_phi: 0.0042 | Regression_energy: 0.6116 | Regression_eta: 0.0092 | Regression_pt: 0.6213 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.3086 | Classification_binary: 0.5500 | Regression_cos_phi: 0.3305 | Regression_energy: 0.3122 | Regression_eta: 0.3545 | Regression_pt: 0.3247 | Regression_sin_phi: 0.3647 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.0269s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7093 | max_loss: 0.1630 | max_backward: 1.5185 | max_optimizer_step: 0.1425 | max_tensorboard_logging: 1.0673 | max_tensorboard_flush: 0.0036 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4652.4 | uss_MB: 3413.8 | child_rss_MB: 5576.9 | allocated_MB: 345.1 | reserved_MB: 24762.0 | inactive_split_MB: 259.0 +rank7 [2026-10-28 11:35:32,723] INFO : Step 600/50000 rank7 | Train Loss: 3.6730 | Train Loss Unweighted: 2.1712 | Train Loss Components: Auxiliary: 0.6920 | Cardinality: 0.3403 | Classification: 0.4255 | Classification_binary: 0.4881 | HitGrouping: 0.4148 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6082 | Regression_eta: 0.0080 | Regression_pt: 0.6335 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2431 | Classification_binary: 0.4665 | Regression_cos_phi: 0.3496 | Regression_energy: 0.3428 | Regression_eta: 0.3224 | Regression_pt: 0.3740 | Regression_sin_phi: 0.2843 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 4.1167s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6664 | max_loss: 0.2467 | max_backward: 1.3732 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0534 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1053 | Memory: rss_MB: 4653.2 | uss_MB: 3421.5 | child_rss_MB: 5586.7 | allocated_MB: 344.0 | reserved_MB: 24762.0 | inactive_split_MB: 229.9 +rank7 [2026-10-28 11:39:09,492] INFO : Step 700/50000 rank7 | Train Loss: 3.3161 | Train Loss Unweighted: 1.9436 | Train Loss Components: Auxiliary: 0.6233 | Cardinality: 0.2890 | Classification: 0.3972 | Classification_binary: 0.4028 | HitGrouping: 0.4056 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5626 | Regression_eta: 0.0078 | Regression_pt: 0.5662 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2159 | Classification_binary: 0.3847 | Regression_cos_phi: 0.3060 | Regression_energy: 0.2277 | Regression_eta: 0.2691 | Regression_pt: 0.2416 | Regression_sin_phi: 0.2107 | LR: 1.00e-03 | DataLoad Time: 0.0174s | Model Forward Time: 3.3985s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.5301 | max_loss: 0.1438 | max_backward: 1.0310 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.8289 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4648.9 | uss_MB: 3418.4 | child_rss_MB: 5577.6 | allocated_MB: 342.9 | reserved_MB: 24762.0 | inactive_split_MB: 283.9 +rank7 [2026-10-28 11:42:49,058] INFO : Step 800/50000 rank7 | Train Loss: 3.5393 | Train Loss Unweighted: 2.1514 | Train Loss Components: Auxiliary: 0.6682 | Cardinality: 0.2789 | Classification: 0.4349 | Classification_binary: 0.4666 | HitGrouping: 0.3740 | Regression_cos_phi: 0.0038 | Regression_energy: 0.6096 | Regression_eta: 0.0087 | Regression_pt: 0.6240 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2508 | Classification_binary: 0.4780 | Regression_cos_phi: 0.2827 | Regression_energy: 0.2989 | Regression_eta: 0.2922 | Regression_pt: 0.3181 | Regression_sin_phi: 0.3461 | LR: 9.99e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.6503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.6987 | max_loss: 0.2493 | max_backward: 1.6595 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4617.8 | uss_MB: 3425.4 | child_rss_MB: 5582.0 | allocated_MB: 343.4 | reserved_MB: 24762.0 | inactive_split_MB: 226.0 +rank7 [2026-10-28 11:46:31,921] INFO : Step 900/50000 rank7 | Train Loss: 3.5463 | Train Loss Unweighted: 2.1660 | Train Loss Components: Auxiliary: 0.6851 | Cardinality: 0.3128 | Classification: 0.4231 | Classification_binary: 0.5000 | HitGrouping: 0.3433 | Regression_cos_phi: 0.0039 | Regression_energy: 0.6113 | Regression_eta: 0.0079 | Regression_pt: 0.6161 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2354 | Classification_binary: 0.4900 | Regression_cos_phi: 0.3016 | Regression_energy: 0.2547 | Regression_eta: 0.2989 | Regression_pt: 0.2647 | Regression_sin_phi: 0.2419 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5691s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0075 | max_forward: 0.6885 | max_loss: 0.4141 | max_backward: 1.4094 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0046 | Memory: rss_MB: 4569.3 | uss_MB: 3425.2 | child_rss_MB: 5584.8 | allocated_MB: 347.5 | reserved_MB: 24762.0 | inactive_split_MB: 268.0 +rank7 [2026-10-28 11:50:09,949] INFO : Step 1000/50000 rank7 | Train Loss: 3.0939 | Train Loss Unweighted: 1.9421 | Train Loss Components: Auxiliary: 0.6141 | Cardinality: 0.3049 | Classification: 0.3605 | Classification_binary: 0.4249 | HitGrouping: 0.3368 | Regression_cos_phi: 0.0026 | Regression_energy: 0.5704 | Regression_eta: 0.0062 | Regression_pt: 0.5745 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2086 | Classification_binary: 0.4273 | Regression_cos_phi: 0.1365 | Regression_energy: 0.2928 | Regression_eta: 0.2145 | Regression_pt: 0.3016 | Regression_sin_phi: 0.2122 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.4166s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6755 | max_loss: 0.1442 | max_backward: 1.4265 | max_optimizer_step: 0.1452 | max_tensorboard_logging: 0.5098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4565.1 | uss_MB: 3424.4 | child_rss_MB: 5589.6 | allocated_MB: 343.7 | reserved_MB: 24762.0 | inactive_split_MB: 254.0 +rank7 [2026-10-28 11:50:10,911] INFO : Running validation on rank7 +rank7 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 11:50:10,911] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 11:50:58,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:50:59,086] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 11:51:19,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 11:51:20,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 11:51:21,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 11:51:22,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 11:51:32,374] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 11:51:32,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:51:33,313] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 11:51:49,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 11:51:50,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 11:51:51,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 11:51:52,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 11:52:02,373] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 11:52:02,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:52:03,469] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 11:52:18,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 11:52:18,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 11:52:19,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 11:52:20,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_1000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 11:52:30,134] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 11:56:43,807] INFO : Step 1100/50000 rank7 | Train Loss: 3.3629 | Train Loss Unweighted: 2.0608 | Train Loss Components: Auxiliary: 0.6489 | Cardinality: 0.3058 | Classification: 0.3696 | Classification_binary: 0.4223 | HitGrouping: 0.3524 | Regression_cos_phi: 0.0036 | Regression_energy: 0.6310 | Regression_eta: 0.0077 | Regression_pt: 0.6232 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2407 | Classification_binary: 0.4124 | Regression_cos_phi: 0.3358 | Regression_energy: 0.2962 | Regression_eta: 0.3212 | Regression_pt: 0.3038 | Regression_sin_phi: 0.2159 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9232s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8667 | max_loss: 0.1529 | max_backward: 1.3522 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 1.0437 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 6286.4 | uss_MB: 5166.3 | child_rss_MB: 5640.4 | allocated_MB: 343.4 | reserved_MB: 23988.0 | inactive_split_MB: 236.6 +rank7 [2026-10-28 12:00:30,891] INFO : Step 1200/50000 rank7 | Train Loss: 3.5831 | Train Loss Unweighted: 2.0592 | Train Loss Components: Auxiliary: 0.6735 | Cardinality: 0.4415 | Classification: 0.4019 | Classification_binary: 0.4990 | HitGrouping: 0.3411 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5709 | Regression_eta: 0.0071 | Regression_pt: 0.5727 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2275 | Classification_binary: 0.4906 | Regression_cos_phi: 0.2820 | Regression_energy: 0.2356 | Regression_eta: 0.2123 | Regression_pt: 0.2458 | Regression_sin_phi: 0.2678 | LR: 9.99e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3986s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7268 | max_loss: 0.4361 | max_backward: 1.5097 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.9829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 6271.7 | uss_MB: 5165.9 | child_rss_MB: 5627.9 | allocated_MB: 348.9 | reserved_MB: 25978.0 | inactive_split_MB: 305.5 +rank7 [2026-10-28 12:04:26,453] INFO : Step 1300/50000 rank7 | Train Loss: 3.1549 | Train Loss Unweighted: 1.9036 | Train Loss Components: Auxiliary: 0.6037 | Cardinality: 0.3060 | Classification: 0.3550 | Classification_binary: 0.4359 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5472 | Regression_eta: 0.0070 | Regression_pt: 0.5521 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2536 | Classification_binary: 0.4762 | Regression_cos_phi: 0.2846 | Regression_energy: 0.2608 | Regression_eta: 0.2453 | Regression_pt: 0.2776 | Regression_sin_phi: 0.2357 | LR: 9.98e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1273s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.6761 | max_loss: 0.2077 | max_backward: 1.4598 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.9420 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5376.3 | uss_MB: 4274.3 | child_rss_MB: 5644.0 | allocated_MB: 345.5 | reserved_MB: 25978.0 | inactive_split_MB: 177.8 +rank7 [2026-10-28 12:08:10,852] INFO : Step 1400/50000 rank7 | Train Loss: 3.0952 | Train Loss Unweighted: 1.8796 | Train Loss Components: Auxiliary: 0.5897 | Cardinality: 0.2807 | Classification: 0.3716 | Classification_binary: 0.3986 | HitGrouping: 0.3244 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5484 | Regression_eta: 0.0066 | Regression_pt: 0.5476 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2269 | Classification_binary: 0.3955 | Regression_cos_phi: 0.3205 | Regression_energy: 0.2301 | Regression_eta: 0.2200 | Regression_pt: 0.2409 | Regression_sin_phi: 0.2080 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7544s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6054 | max_loss: 0.1502 | max_backward: 1.2375 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.9880 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5378.8 | uss_MB: 4275.3 | child_rss_MB: 5642.4 | allocated_MB: 344.7 | reserved_MB: 25978.0 | inactive_split_MB: 245.1 +rank7 [2026-10-28 12:11:57,637] INFO : Step 1500/50000 rank7 | Train Loss: 3.1840 | Train Loss Unweighted: 1.9434 | Train Loss Components: Auxiliary: 0.6053 | Cardinality: 0.3175 | Classification: 0.3517 | Classification_binary: 0.4488 | HitGrouping: 0.3281 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5615 | Regression_eta: 0.0062 | Regression_pt: 0.5686 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2172 | Classification_binary: 0.4426 | Regression_cos_phi: 0.2172 | Regression_energy: 0.2867 | Regression_eta: 0.2123 | Regression_pt: 0.2987 | Regression_sin_phi: 0.2753 | LR: 9.98e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5810s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0072 | max_forward: 0.7916 | max_loss: 0.2324 | max_backward: 2.0000 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6692 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 5377.6 | uss_MB: 4100.4 | child_rss_MB: 5620.0 | allocated_MB: 346.7 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank7 [2026-10-28 12:15:43,592] INFO : Step 1600/50000 rank7 | Train Loss: 3.1589 | Train Loss Unweighted: 1.9281 | Train Loss Components: Auxiliary: 0.6035 | Cardinality: 0.3021 | Classification: 0.3610 | Classification_binary: 0.4569 | HitGrouping: 0.3218 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5451 | Regression_eta: 0.0063 | Regression_pt: 0.5522 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2074 | Classification_binary: 0.4355 | Regression_cos_phi: 0.2602 | Regression_energy: 0.2673 | Regression_eta: 0.2041 | Regression_pt: 0.2786 | Regression_sin_phi: 0.1958 | LR: 9.98e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.0363s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0787 | max_forward: 0.8140 | max_loss: 0.1765 | max_backward: 1.6147 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.6564 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5365.2 | uss_MB: 4280.1 | child_rss_MB: 5645.5 | allocated_MB: 343.6 | reserved_MB: 26998.0 | inactive_split_MB: 260.8 +rank7 [2026-10-28 12:19:37,261] INFO : Step 1700/50000 rank7 | Train Loss: 2.8564 | Train Loss Unweighted: 1.7368 | Train Loss Components: Auxiliary: 0.5447 | Cardinality: 0.2789 | Classification: 0.3063 | Classification_binary: 0.3696 | HitGrouping: 0.3164 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5223 | Regression_eta: 0.0064 | Regression_pt: 0.5266 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2065 | Classification_binary: 0.3445 | Regression_cos_phi: 0.2240 | Regression_energy: 0.2582 | Regression_eta: 0.2013 | Regression_pt: 0.2701 | Regression_sin_phi: 0.1881 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.3918s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6521 | max_loss: 0.4399 | max_backward: 1.2259 | max_optimizer_step: 0.1647 | max_tensorboard_logging: 0.5933 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0094 | Memory: rss_MB: 5365.6 | uss_MB: 4281.7 | child_rss_MB: 5645.2 | allocated_MB: 340.9 | reserved_MB: 26998.0 | inactive_split_MB: 222.6 +rank7 [2026-10-28 12:23:17,460] INFO : Step 1800/50000 rank7 | Train Loss: 2.9137 | Train Loss Unweighted: 1.7870 | Train Loss Components: Auxiliary: 0.5568 | Cardinality: 0.2558 | Classification: 0.3663 | Classification_binary: 0.4122 | HitGrouping: 0.3196 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4940 | Regression_eta: 0.0060 | Regression_pt: 0.5025 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2066 | Classification_binary: 0.4210 | Regression_cos_phi: 0.1765 | Regression_energy: 0.2380 | Regression_eta: 0.1864 | Regression_pt: 0.2515 | Regression_sin_phi: 0.1937 | LR: 9.97e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9408s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.6827 | max_loss: 0.4207 | max_backward: 1.4187 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 1.0036 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0113 | Memory: rss_MB: 5366.9 | uss_MB: 4284.0 | child_rss_MB: 5649.8 | allocated_MB: 345.3 | reserved_MB: 26998.0 | inactive_split_MB: 254.7 +rank7 [2026-10-28 12:27:10,966] INFO : Step 1900/50000 rank7 | Train Loss: 3.2174 | Train Loss Unweighted: 1.9616 | Train Loss Components: Auxiliary: 0.6145 | Cardinality: 0.3223 | Classification: 0.3736 | Classification_binary: 0.4751 | HitGrouping: 0.3275 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5467 | Regression_eta: 0.0063 | Regression_pt: 0.5534 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2227 | Classification_binary: 0.4959 | Regression_cos_phi: 0.2937 | Regression_energy: 0.2793 | Regression_eta: 0.2474 | Regression_pt: 0.2921 | Regression_sin_phi: 0.3261 | LR: 9.97e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.5693s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0069 | max_forward: 0.9162 | max_loss: 0.2517 | max_backward: 1.7898 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 1.1141 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0095 | Memory: rss_MB: 5363.9 | uss_MB: 4285.1 | child_rss_MB: 5613.4 | allocated_MB: 345.8 | reserved_MB: 26998.0 | inactive_split_MB: 272.6 +rank7 [2026-10-28 12:30:59,046] INFO : Step 2000/50000 rank7 | Train Loss: 2.8430 | Train Loss Unweighted: 1.6812 | Train Loss Components: Auxiliary: 0.5343 | Cardinality: 0.2835 | Classification: 0.3225 | Classification_binary: 0.3920 | HitGrouping: 0.3167 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4759 | Regression_eta: 0.0061 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2019 | Classification_binary: 0.4167 | Regression_cos_phi: 0.2327 | Regression_energy: 0.2517 | Regression_eta: 0.2511 | Regression_pt: 0.2577 | Regression_sin_phi: 0.2916 | LR: 9.96e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.1419s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0560 | max_forward: 0.7119 | max_loss: 0.1480 | max_backward: 1.5407 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 1.0046 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 5364.0 | uss_MB: 4287.8 | child_rss_MB: 5645.6 | allocated_MB: 341.2 | reserved_MB: 26998.0 | inactive_split_MB: 244.1 +rank7 [2026-10-28 12:31:00,064] INFO : Running validation on rank7 +rank7 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 12:31:00,064] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 12:31:47,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:31:48,296] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 12:32:13,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:32:14,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:32:15,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:32:16,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:32:26,482] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 12:32:27,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:32:27,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 12:32:44,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:32:45,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:32:45,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:32:46,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:32:56,975] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:32:57,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:32:57,641] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 12:33:13,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:33:13,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:33:14,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:33:15,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_2000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:33:25,109] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 12:37:32,605] INFO : Step 2100/50000 rank7 | Train Loss: 2.8273 | Train Loss Unweighted: 1.7682 | Train Loss Components: Auxiliary: 0.5439 | Cardinality: 0.2430 | Classification: 0.3397 | Classification_binary: 0.3685 | HitGrouping: 0.3139 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5195 | Regression_eta: 0.0062 | Regression_pt: 0.5288 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2202 | Classification_binary: 0.4104 | Regression_cos_phi: 0.2465 | Regression_energy: 0.2338 | Regression_eta: 0.2217 | Regression_pt: 0.2464 | Regression_sin_phi: 0.2203 | LR: 9.96e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7738s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6299 | max_loss: 0.1401 | max_backward: 1.1798 | max_optimizer_step: 0.1721 | max_tensorboard_logging: 1.0001 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 6269.2 | uss_MB: 5190.6 | child_rss_MB: 5665.2 | allocated_MB: 345.0 | reserved_MB: 22938.0 | inactive_split_MB: 264.2 +rank7 [2026-10-28 12:41:23,028] INFO : Step 2200/50000 rank7 | Train Loss: 3.2084 | Train Loss Unweighted: 1.9060 | Train Loss Components: Auxiliary: 0.6015 | Cardinality: 0.3226 | Classification: 0.3562 | Classification_binary: 0.4497 | HitGrouping: 0.3191 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5388 | Regression_eta: 0.0073 | Regression_pt: 0.5472 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2163 | Classification_binary: 0.4682 | Regression_cos_phi: 0.2577 | Regression_energy: 0.2547 | Regression_eta: 0.2329 | Regression_pt: 0.2652 | Regression_sin_phi: 0.2554 | LR: 9.96e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7067s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7140 | max_loss: 0.1880 | max_backward: 1.5011 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.4995 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6262.4 | uss_MB: 5191.7 | child_rss_MB: 5669.9 | allocated_MB: 345.6 | reserved_MB: 23940.0 | inactive_split_MB: 206.2 +rank7 [2026-10-28 12:45:14,506] INFO : Step 2300/50000 rank7 | Train Loss: 3.0003 | Train Loss Unweighted: 1.7680 | Train Loss Components: Auxiliary: 0.5700 | Cardinality: 0.3096 | Classification: 0.3360 | Classification_binary: 0.4059 | HitGrouping: 0.3123 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5040 | Regression_eta: 0.0062 | Regression_pt: 0.5094 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1621 | Classification_binary: 0.3794 | Regression_cos_phi: 0.2266 | Regression_energy: 0.2330 | Regression_eta: 0.2072 | Regression_pt: 0.2432 | Regression_sin_phi: 0.2146 | LR: 9.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9865s | Phase Times: max_pre_step_barrier: 0.0116 | max_batch_to_device: 0.1146 | max_forward: 1.5244 | max_loss: 0.2917 | max_backward: 2.3519 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.7022 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5487.5 | uss_MB: 4415.2 | child_rss_MB: 5649.8 | allocated_MB: 345.0 | reserved_MB: 24946.0 | inactive_split_MB: 294.6 +rank7 [2026-10-28 12:49:08,621] INFO : Step 2400/50000 rank7 | Train Loss: 2.7199 | Train Loss Unweighted: 1.6362 | Train Loss Components: Auxiliary: 0.5196 | Cardinality: 0.2394 | Classification: 0.3003 | Classification_binary: 0.3530 | HitGrouping: 0.3118 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4839 | Regression_eta: 0.0061 | Regression_pt: 0.4874 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2060 | Classification_binary: 0.3431 | Regression_cos_phi: 0.2986 | Regression_energy: 0.2422 | Regression_eta: 0.2108 | Regression_pt: 0.2530 | Regression_sin_phi: 0.2340 | LR: 9.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4491s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.6357 | max_loss: 0.1398 | max_backward: 1.3847 | max_optimizer_step: 0.1643 | max_tensorboard_logging: 0.5272 | max_tensorboard_flush: 0.0234 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5490.8 | uss_MB: 4419.6 | child_rss_MB: 5643.5 | allocated_MB: 339.8 | reserved_MB: 24946.0 | inactive_split_MB: 274.2 +rank7 [2026-10-28 12:52:58,131] INFO : Step 2500/50000 rank7 | Train Loss: 3.0955 | Train Loss Unweighted: 1.8742 | Train Loss Components: Auxiliary: 0.5946 | Cardinality: 0.2885 | Classification: 0.3830 | Classification_binary: 0.4355 | HitGrouping: 0.3199 | Regression_cos_phi: 0.0032 | Regression_energy: 0.5173 | Regression_eta: 0.0063 | Regression_pt: 0.5255 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1978 | Classification_binary: 0.4183 | Regression_cos_phi: 0.2359 | Regression_energy: 0.2506 | Regression_eta: 0.2226 | Regression_pt: 0.2568 | Regression_sin_phi: 0.2022 | LR: 9.94e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3987s | Phase Times: max_pre_step_barrier: 0.0131 | max_batch_to_device: 0.0067 | max_forward: 0.7642 | max_loss: 0.2300 | max_backward: 1.7253 | max_optimizer_step: 0.1645 | max_tensorboard_logging: 1.0449 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5494.7 | uss_MB: 4424.8 | child_rss_MB: 5623.1 | allocated_MB: 346.8 | reserved_MB: 25106.0 | inactive_split_MB: 272.2 +rank7 [2026-10-28 12:56:45,818] INFO : Step 2600/50000 rank7 | Train Loss: 2.9843 | Train Loss Unweighted: 1.8235 | Train Loss Components: Auxiliary: 0.5739 | Cardinality: 0.2748 | Classification: 0.3335 | Classification_binary: 0.4358 | HitGrouping: 0.3134 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5178 | Regression_eta: 0.0062 | Regression_pt: 0.5241 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1900 | Classification_binary: 0.3931 | Regression_cos_phi: 0.2274 | Regression_energy: 0.2479 | Regression_eta: 0.2102 | Regression_pt: 0.2622 | Regression_sin_phi: 0.1883 | LR: 9.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2542s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7380 | max_loss: 0.4090 | max_backward: 1.5929 | max_optimizer_step: 0.1652 | max_tensorboard_logging: 0.9150 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 5492.1 | uss_MB: 4425.7 | child_rss_MB: 5640.6 | allocated_MB: 351.1 | reserved_MB: 25106.0 | inactive_split_MB: 270.9 +rank7 [2026-10-28 13:00:38,244] INFO : Step 2700/50000 rank7 | Train Loss: 2.6756 | Train Loss Unweighted: 1.6123 | Train Loss Components: Auxiliary: 0.5146 | Cardinality: 0.2532 | Classification: 0.2885 | Classification_binary: 0.3390 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4849 | Regression_eta: 0.0056 | Regression_pt: 0.4888 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1711 | Classification_binary: 0.3262 | Regression_cos_phi: 0.1968 | Regression_energy: 0.2248 | Regression_eta: 0.1635 | Regression_pt: 0.2310 | Regression_sin_phi: 0.1679 | LR: 9.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0885s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.6766 | max_loss: 0.4046 | max_backward: 1.4172 | max_optimizer_step: 0.1640 | max_tensorboard_logging: 1.0228 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5492.1 | uss_MB: 4426.5 | child_rss_MB: 5641.3 | allocated_MB: 344.3 | reserved_MB: 25610.0 | inactive_split_MB: 219.7 +rank7 [2026-10-28 13:04:27,895] INFO : Step 2800/50000 rank7 | Train Loss: 2.7804 | Train Loss Unweighted: 1.6613 | Train Loss Components: Auxiliary: 0.5226 | Cardinality: 0.2371 | Classification: 0.3098 | Classification_binary: 0.3738 | HitGrouping: 0.3122 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4778 | Regression_eta: 0.0058 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1997 | Classification_binary: 0.4192 | Regression_cos_phi: 0.2755 | Regression_energy: 0.2202 | Regression_eta: 0.2479 | Regression_pt: 0.2333 | Regression_sin_phi: 0.2437 | LR: 9.93e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1949s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1126 | max_forward: 0.7778 | max_loss: 0.1585 | max_backward: 1.5677 | max_optimizer_step: 0.1611 | max_tensorboard_logging: 1.0022 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5491.8 | uss_MB: 4428.9 | child_rss_MB: 5639.8 | allocated_MB: 344.7 | reserved_MB: 26648.0 | inactive_split_MB: 272.7 +rank7 [2026-10-28 13:08:19,435] INFO : Step 2900/50000 rank7 | Train Loss: 3.1351 | Train Loss Unweighted: 1.8024 | Train Loss Components: Auxiliary: 0.5764 | Cardinality: 0.3201 | Classification: 0.3385 | Classification_binary: 0.4395 | HitGrouping: 0.3160 | Regression_cos_phi: 0.0035 | Regression_energy: 0.5010 | Regression_eta: 0.0074 | Regression_pt: 0.5091 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1954 | Classification_binary: 0.4484 | Regression_cos_phi: 0.2540 | Regression_energy: 0.2664 | Regression_eta: 0.2563 | Regression_pt: 0.2813 | Regression_sin_phi: 0.2617 | LR: 9.93e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1925s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.8438 | max_loss: 0.4589 | max_backward: 1.6595 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.6117 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5490.7 | uss_MB: 4429.4 | child_rss_MB: 5641.9 | allocated_MB: 345.7 | reserved_MB: 26648.0 | inactive_split_MB: 292.2 +rank7 [2026-10-28 13:12:09,608] INFO : Step 3000/50000 rank7 | Train Loss: 2.7668 | Train Loss Unweighted: 1.6167 | Train Loss Components: Auxiliary: 0.5327 | Cardinality: 0.2887 | Classification: 0.2941 | Classification_binary: 0.3813 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4635 | Regression_eta: 0.0060 | Regression_pt: 0.4664 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1731 | Classification_binary: 0.3716 | Regression_cos_phi: 0.1863 | Regression_energy: 0.2179 | Regression_eta: 0.1897 | Regression_pt: 0.2293 | Regression_sin_phi: 0.1906 | LR: 9.92e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5268s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0061 | max_forward: 0.6427 | max_loss: 0.4141 | max_backward: 1.3155 | max_optimizer_step: 0.1656 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5492.1 | uss_MB: 4430.4 | child_rss_MB: 5641.4 | allocated_MB: 340.6 | reserved_MB: 26648.0 | inactive_split_MB: 243.0 +rank7 [2026-10-28 13:12:10,495] INFO : Running validation on rank7 +rank7 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:12:10,496] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:13:02,888] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:13:03,433] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 13:13:30,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:13:30,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:13:31,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:13:32,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:13:42,929] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:13:43,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:13:44,105] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 13:14:00,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:14:01,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:14:02,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:14:03,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:14:13,284] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:14:13,731] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:14:14,292] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 13:14:29,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:14:30,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:14:31,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:14:31,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_3000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:14:41,722] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 13:19:02,742] INFO : Step 3100/50000 rank7 | Train Loss: 2.5793 | Train Loss Unweighted: 1.5470 | Train Loss Components: Auxiliary: 0.4962 | Cardinality: 0.2351 | Classification: 0.2845 | Classification_binary: 0.3251 | HitGrouping: 0.3089 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4555 | Regression_eta: 0.0050 | Regression_pt: 0.4714 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1548 | Classification_binary: 0.3170 | Regression_cos_phi: 0.2205 | Regression_energy: 0.2328 | Regression_eta: 0.1687 | Regression_pt: 0.2471 | Regression_sin_phi: 0.2077 | LR: 9.91e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.5830s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 1.8691 | max_loss: 0.7036 | max_backward: 2.9619 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.8405 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5605.1 | uss_MB: 4538.1 | child_rss_MB: 5660.0 | allocated_MB: 344.1 | reserved_MB: 24074.0 | inactive_split_MB: 274.7 +rank7 [2026-10-28 13:23:00,673] INFO : Step 3200/50000 rank7 | Train Loss: 3.0181 | Train Loss Unweighted: 1.7926 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2629 | Classification: 0.3449 | Classification_binary: 0.4274 | HitGrouping: 0.3146 | Regression_cos_phi: 0.0034 | Regression_energy: 0.4997 | Regression_eta: 0.0066 | Regression_pt: 0.5072 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2036 | Classification_binary: 0.4557 | Regression_cos_phi: 0.2725 | Regression_energy: 0.2486 | Regression_eta: 0.2469 | Regression_pt: 0.2611 | Regression_sin_phi: 0.2355 | LR: 9.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8528s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7530 | max_loss: 0.4451 | max_backward: 1.5834 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5865 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 5614.0 | uss_MB: 4547.2 | child_rss_MB: 5662.7 | allocated_MB: 346.2 | reserved_MB: 24876.0 | inactive_split_MB: 243.8 +rank7 [2026-10-28 13:26:54,112] INFO : Step 3300/50000 rank7 | Train Loss: 2.7445 | Train Loss Unweighted: 1.6254 | Train Loss Components: Auxiliary: 0.5135 | Cardinality: 0.2318 | Classification: 0.3098 | Classification_binary: 0.4018 | HitGrouping: 0.3072 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4497 | Regression_eta: 0.0063 | Regression_pt: 0.4518 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.2002 | Classification_binary: 0.4196 | Regression_cos_phi: 0.2696 | Regression_energy: 0.2274 | Regression_eta: 0.2984 | Regression_pt: 0.2369 | Regression_sin_phi: 0.2247 | LR: 9.90e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7311s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7226 | max_loss: 0.1486 | max_backward: 1.5893 | max_optimizer_step: 0.1737 | max_tensorboard_logging: 0.4833 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5608.1 | uss_MB: 4548.0 | child_rss_MB: 5659.1 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 269.6 +rank7 [2026-10-28 13:30:48,597] INFO : Step 3400/50000 rank7 | Train Loss: 2.5851 | Train Loss Unweighted: 1.5750 | Train Loss Components: Auxiliary: 0.5090 | Cardinality: 0.2404 | Classification: 0.2672 | Classification_binary: 0.3141 | HitGrouping: 0.3004 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4916 | Regression_eta: 0.0049 | Regression_pt: 0.4918 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1410 | Classification_binary: 0.3173 | Regression_cos_phi: 0.1926 | Regression_energy: 0.2426 | Regression_eta: 0.1480 | Regression_pt: 0.2511 | Regression_sin_phi: 0.2123 | LR: 9.90e-04 | DataLoad Time: 0.0132s | Model Forward Time: 4.0690s | Phase Times: max_pre_step_barrier: 0.1039 | max_batch_to_device: 0.0076 | max_forward: 1.0536 | max_loss: 0.4553 | max_backward: 1.6426 | max_optimizer_step: 0.1711 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5606.2 | uss_MB: 4546.5 | child_rss_MB: 5665.0 | allocated_MB: 342.4 | reserved_MB: 25058.0 | inactive_split_MB: 205.5 +rank7 [2026-10-28 13:34:47,772] INFO : Step 3500/50000 rank7 | Train Loss: 2.6457 | Train Loss Unweighted: 1.5989 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2123 | Classification: 0.2995 | Classification_binary: 0.3796 | HitGrouping: 0.3058 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4509 | Regression_eta: 0.0053 | Regression_pt: 0.4580 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1843 | Classification_binary: 0.3784 | Regression_cos_phi: 0.2169 | Regression_energy: 0.2351 | Regression_eta: 0.1465 | Regression_pt: 0.2463 | Regression_sin_phi: 0.2170 | LR: 9.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6683s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.6945 | max_loss: 0.4230 | max_backward: 1.5280 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5613.4 | uss_MB: 4555.2 | child_rss_MB: 5661.3 | allocated_MB: 347.1 | reserved_MB: 25892.0 | inactive_split_MB: 216.1 +rank7 [2026-10-28 13:38:46,122] INFO : Step 3600/50000 rank7 | Train Loss: 2.8182 | Train Loss Unweighted: 1.6075 | Train Loss Components: Auxiliary: 0.5171 | Cardinality: 0.2811 | Classification: 0.2850 | Classification_binary: 0.4412 | HitGrouping: 0.3082 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4310 | Regression_eta: 0.0059 | Regression_pt: 0.4382 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1908 | Classification_binary: 0.4277 | Regression_cos_phi: 0.2218 | Regression_energy: 0.2217 | Regression_eta: 0.2118 | Regression_pt: 0.2321 | Regression_sin_phi: 0.2258 | LR: 9.89e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.1024s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0512 | max_forward: 1.0921 | max_loss: 0.1707 | max_backward: 1.7417 | max_optimizer_step: 0.1650 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 5606.3 | uss_MB: 4552.7 | child_rss_MB: 5662.8 | allocated_MB: 345.6 | reserved_MB: 25892.0 | inactive_split_MB: 256.7 +rank7 [2026-10-28 13:42:35,794] INFO : Step 3700/50000 rank7 | Train Loss: 2.5522 | Train Loss Unweighted: 1.5184 | Train Loss Components: Auxiliary: 0.4796 | Cardinality: 0.2005 | Classification: 0.2644 | Classification_binary: 0.3442 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4510 | Regression_eta: 0.0059 | Regression_pt: 0.4474 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1502 | Classification_binary: 0.3688 | Regression_cos_phi: 0.2209 | Regression_energy: 0.2020 | Regression_eta: 0.2060 | Regression_pt: 0.2102 | Regression_sin_phi: 0.2279 | LR: 9.88e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2041s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6918 | max_loss: 0.2343 | max_backward: 1.5469 | max_optimizer_step: 0.1727 | max_tensorboard_logging: 1.0574 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0192 | Memory: rss_MB: 5608.3 | uss_MB: 4555.0 | child_rss_MB: 5652.9 | allocated_MB: 343.6 | reserved_MB: 26092.0 | inactive_split_MB: 305.3 +rank7 [2026-10-28 13:46:34,311] INFO : Step 3800/50000 rank7 | Train Loss: 2.6070 | Train Loss Unweighted: 1.5355 | Train Loss Components: Auxiliary: 0.4996 | Cardinality: 0.2241 | Classification: 0.2828 | Classification_binary: 0.3541 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4390 | Regression_eta: 0.0054 | Regression_pt: 0.4486 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1895 | Classification_binary: 0.3733 | Regression_cos_phi: 0.2254 | Regression_energy: 0.2116 | Regression_eta: 0.2101 | Regression_pt: 0.2250 | Regression_sin_phi: 0.2734 | LR: 9.87e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9869s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8774 | max_loss: 0.2471 | max_backward: 1.2863 | max_optimizer_step: 0.1697 | max_tensorboard_logging: 0.8925 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5605.7 | uss_MB: 4553.9 | child_rss_MB: 5644.9 | allocated_MB: 343.9 | reserved_MB: 26092.0 | inactive_split_MB: 170.1 +rank7 [2026-10-28 13:50:23,157] INFO : Step 3900/50000 rank7 | Train Loss: 2.8043 | Train Loss Unweighted: 1.6477 | Train Loss Components: Auxiliary: 0.5326 | Cardinality: 0.2637 | Classification: 0.3011 | Classification_binary: 0.4355 | HitGrouping: 0.3108 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4472 | Regression_eta: 0.0057 | Regression_pt: 0.4524 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1783 | Classification_binary: 0.4573 | Regression_cos_phi: 0.2144 | Regression_energy: 0.2002 | Regression_eta: 0.2082 | Regression_pt: 0.2104 | Regression_sin_phi: 0.2337 | LR: 9.87e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2427s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.7761 | max_loss: 0.4335 | max_backward: 1.5749 | max_optimizer_step: 0.2392 | max_tensorboard_logging: 0.8109 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5600.6 | uss_MB: 4559.4 | child_rss_MB: 5620.4 | allocated_MB: 346.8 | reserved_MB: 27926.0 | inactive_split_MB: 291.8 +rank7 [2026-10-28 13:54:11,924] INFO : Step 4000/50000 rank7 | Train Loss: 2.4937 | Train Loss Unweighted: 1.5292 | Train Loss Components: Auxiliary: 0.4882 | Cardinality: 0.1891 | Classification: 0.2584 | Classification_binary: 0.3612 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4462 | Regression_eta: 0.0046 | Regression_pt: 0.4536 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1483 | Classification_binary: 0.3035 | Regression_cos_phi: 0.2214 | Regression_energy: 0.2315 | Regression_eta: 0.1497 | Regression_pt: 0.2455 | Regression_sin_phi: 0.1895 | LR: 9.86e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5792s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6447 | max_loss: 0.2490 | max_backward: 1.3853 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.5675 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5600.8 | uss_MB: 4559.8 | child_rss_MB: 5611.2 | allocated_MB: 341.3 | reserved_MB: 27926.0 | inactive_split_MB: 187.3 +rank7 [2026-10-28 13:54:12,844] INFO : Running validation on rank7 +rank7 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:54:12,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:55:03,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:55:03,607] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 13:55:33,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:55:34,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:55:35,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:55:36,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:55:46,275] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:55:46,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:55:46,898] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 13:56:03,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:56:04,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:56:05,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:56:06,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:56:16,022] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:56:16,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:56:16,990] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 13:56:32,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:56:32,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:56:33,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:56:34,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_4000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:56:44,060] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 14:00:56,374] INFO : Step 4100/50000 rank7 | Train Loss: 2.5304 | Train Loss Unweighted: 1.5408 | Train Loss Components: Auxiliary: 0.4998 | Cardinality: 0.2043 | Classification: 0.2670 | Classification_binary: 0.3272 | HitGrouping: 0.3035 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4679 | Regression_eta: 0.0051 | Regression_pt: 0.4683 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1454 | Classification_binary: 0.3442 | Regression_cos_phi: 0.2067 | Regression_energy: 0.2165 | Regression_eta: 0.1569 | Regression_pt: 0.2243 | Regression_sin_phi: 0.1696 | LR: 9.85e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9525s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.6514 | max_loss: 0.3976 | max_backward: 1.3587 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8965 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6741.3 | uss_MB: 5697.8 | child_rss_MB: 5621.2 | allocated_MB: 342.6 | reserved_MB: 23310.0 | inactive_split_MB: 192.0 +rank7 [2026-10-28 14:04:50,729] INFO : Step 4200/50000 rank7 | Train Loss: 2.7631 | Train Loss Unweighted: 1.6490 | Train Loss Components: Auxiliary: 0.5289 | Cardinality: 0.2415 | Classification: 0.3169 | Classification_binary: 0.3855 | HitGrouping: 0.3034 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4651 | Regression_eta: 0.0060 | Regression_pt: 0.4696 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.4186 | Regression_cos_phi: 0.2386 | Regression_energy: 0.2138 | Regression_eta: 0.2178 | Regression_pt: 0.2223 | Regression_sin_phi: 0.2316 | LR: 9.84e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4863s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 0.9508 | max_loss: 0.1822 | max_backward: 1.8934 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.5999 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5703.3 | uss_MB: 4641.3 | child_rss_MB: 5616.2 | allocated_MB: 346.1 | reserved_MB: 25722.0 | inactive_split_MB: 282.0 +rank7 [2026-10-28 14:08:42,279] INFO : Step 4300/50000 rank7 | Train Loss: 2.6756 | Train Loss Unweighted: 1.5502 | Train Loss Components: Auxiliary: 0.5064 | Cardinality: 0.2365 | Classification: 0.2806 | Classification_binary: 0.3918 | HitGrouping: 0.3031 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4306 | Regression_eta: 0.0055 | Regression_pt: 0.4358 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1639 | Classification_binary: 0.3928 | Regression_cos_phi: 0.2283 | Regression_energy: 0.2120 | Regression_eta: 0.1441 | Regression_pt: 0.2189 | Regression_sin_phi: 0.2447 | LR: 9.84e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1062s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0067 | max_forward: 0.9457 | max_loss: 0.4060 | max_backward: 1.4811 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 5693.1 | uss_MB: 4571.5 | child_rss_MB: 5598.2 | allocated_MB: 343.2 | reserved_MB: 26732.0 | inactive_split_MB: 251.1 +rank7 [2026-10-28 14:12:29,194] INFO : Step 4400/50000 rank7 | Train Loss: 2.3109 | Train Loss Unweighted: 1.3672 | Train Loss Components: Auxiliary: 0.4496 | Cardinality: 0.1986 | Classification: 0.2331 | Classification_binary: 0.2990 | HitGrouping: 0.2941 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4132 | Regression_eta: 0.0047 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.2812 | Regression_cos_phi: 0.1536 | Regression_energy: 0.1906 | Regression_eta: 0.1670 | Regression_pt: 0.1977 | Regression_sin_phi: 0.2151 | LR: 9.83e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.7773s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6353 | max_loss: 0.4138 | max_backward: 1.3887 | max_optimizer_step: 0.1827 | max_tensorboard_logging: 0.5742 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.2583 | Memory: rss_MB: 5659.5 | uss_MB: 4662.0 | child_rss_MB: 5445.2 | allocated_MB: 341.5 | reserved_MB: 26732.0 | inactive_split_MB: 170.0 +rank7 [2026-10-28 14:16:26,829] INFO : Step 4500/50000 rank7 | Train Loss: 2.5984 | Train Loss Unweighted: 1.5195 | Train Loss Components: Auxiliary: 0.4987 | Cardinality: 0.2471 | Classification: 0.2774 | Classification_binary: 0.3569 | HitGrouping: 0.3060 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4339 | Regression_eta: 0.0053 | Regression_pt: 0.4406 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1689 | Classification_binary: 0.3731 | Regression_cos_phi: 0.2343 | Regression_energy: 0.2015 | Regression_eta: 0.1705 | Regression_pt: 0.2168 | Regression_sin_phi: 0.1838 | LR: 9.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.4690s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6495 | max_loss: 0.2180 | max_backward: 1.3518 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5657.0 | uss_MB: 4664.6 | child_rss_MB: 5423.9 | allocated_MB: 343.9 | reserved_MB: 26732.0 | inactive_split_MB: 206.1 +rank7 [2026-10-28 14:20:31,827] INFO : Step 4600/50000 rank7 | Train Loss: 2.7184 | Train Loss Unweighted: 1.5593 | Train Loss Components: Auxiliary: 0.5081 | Cardinality: 0.2774 | Classification: 0.2860 | Classification_binary: 0.4236 | HitGrouping: 0.2985 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4170 | Regression_eta: 0.0056 | Regression_pt: 0.4214 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1671 | Classification_binary: 0.4282 | Regression_cos_phi: 0.2554 | Regression_energy: 0.2039 | Regression_eta: 0.1910 | Regression_pt: 0.2141 | Regression_sin_phi: 0.2595 | LR: 9.81e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.1376s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7879 | max_loss: 0.4433 | max_backward: 1.6723 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5814 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5661.2 | uss_MB: 4667.5 | child_rss_MB: 5429.6 | allocated_MB: 344.9 | reserved_MB: 26732.0 | inactive_split_MB: 286.0 +rank7 [2026-10-28 14:24:22,133] INFO : Step 4700/50000 rank7 | Train Loss: 2.3807 | Train Loss Unweighted: 1.3590 | Train Loss Components: Auxiliary: 0.4626 | Cardinality: 0.2575 | Classification: 0.2241 | Classification_binary: 0.3309 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3953 | Regression_eta: 0.0045 | Regression_pt: 0.3995 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1640 | Classification_binary: 0.3203 | Regression_cos_phi: 0.1340 | Regression_energy: 0.2246 | Regression_eta: 0.1299 | Regression_pt: 0.2377 | Regression_sin_phi: 0.2128 | LR: 9.81e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3023s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0065 | max_forward: 0.8174 | max_loss: 0.4089 | max_backward: 1.8722 | max_optimizer_step: 0.1690 | max_tensorboard_logging: 0.4948 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5656.5 | uss_MB: 4668.7 | child_rss_MB: 5406.7 | allocated_MB: 343.4 | reserved_MB: 27766.0 | inactive_split_MB: 200.0 +rank7 [2026-10-28 14:28:09,261] INFO : Step 4800/50000 rank7 | Train Loss: 2.5274 | Train Loss Unweighted: 1.4371 | Train Loss Components: Auxiliary: 0.4731 | Cardinality: 0.2264 | Classification: 0.2668 | Classification_binary: 0.3338 | HitGrouping: 0.2992 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4099 | Regression_eta: 0.0053 | Regression_pt: 0.4156 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1523 | Classification_binary: 0.3043 | Regression_cos_phi: 0.2478 | Regression_energy: 0.1983 | Regression_eta: 0.1911 | Regression_pt: 0.2038 | Regression_sin_phi: 0.2232 | LR: 9.80e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9651s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8331 | max_loss: 0.4098 | max_backward: 1.6670 | max_optimizer_step: 0.1676 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0944 | Memory: rss_MB: 5658.1 | uss_MB: 4671.2 | child_rss_MB: 5402.9 | allocated_MB: 344.2 | reserved_MB: 27766.0 | inactive_split_MB: 198.8 +rank7 [2026-10-28 14:32:01,138] INFO : Step 4900/50000 rank7 | Train Loss: 2.7073 | Train Loss Unweighted: 1.5458 | Train Loss Components: Auxiliary: 0.5069 | Cardinality: 0.2542 | Classification: 0.2921 | Classification_binary: 0.4067 | HitGrouping: 0.2979 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4144 | Regression_eta: 0.0059 | Regression_pt: 0.4207 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1403 | Classification_binary: 0.3665 | Regression_cos_phi: 0.1937 | Regression_energy: 0.1958 | Regression_eta: 0.1647 | Regression_pt: 0.2043 | Regression_sin_phi: 0.1960 | LR: 9.79e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8456s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9395 | max_loss: 0.1830 | max_backward: 2.0948 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.5991 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.3015 | Memory: rss_MB: 5659.9 | uss_MB: 4600.4 | child_rss_MB: 5403.5 | allocated_MB: 345.1 | reserved_MB: 27766.0 | inactive_split_MB: 252.0 +rank7 [2026-10-28 14:35:52,796] INFO : Step 5000/50000 rank7 | Train Loss: 2.5353 | Train Loss Unweighted: 1.4509 | Train Loss Components: Auxiliary: 0.4927 | Cardinality: 0.2801 | Classification: 0.2561 | Classification_binary: 0.3518 | HitGrouping: 0.2886 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4141 | Regression_eta: 0.0049 | Regression_pt: 0.4188 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1448 | Classification_binary: 0.3393 | Regression_cos_phi: 0.1547 | Regression_energy: 0.1956 | Regression_eta: 0.1350 | Regression_pt: 0.2045 | Regression_sin_phi: 0.1598 | LR: 9.78e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8413s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.6995 | max_loss: 0.4197 | max_backward: 1.5268 | max_optimizer_step: 0.1661 | max_tensorboard_logging: 0.6238 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5660.5 | uss_MB: 4673.2 | child_rss_MB: 5404.3 | allocated_MB: 346.0 | reserved_MB: 27766.0 | inactive_split_MB: 268.0 +rank7 [2026-10-28 14:35:53,776] INFO : Running validation on rank7 +rank7 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 14:35:53,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 14:36:42,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:36:43,013] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 14:37:08,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 14:37:09,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 14:37:10,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 14:37:11,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 14:37:21,880] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 14:37:22,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:37:22,910] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 14:37:38,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 14:37:39,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 14:37:40,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 14:37:41,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 14:37:51,670] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 14:37:52,214] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:37:52,791] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 14:38:08,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 14:38:08,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 14:38:09,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 14:38:09,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_5000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 14:38:19,915] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 14:42:36,615] INFO : Step 5100/50000 rank7 | Train Loss: 2.2910 | Train Loss Unweighted: 1.3644 | Train Loss Components: Auxiliary: 0.4541 | Cardinality: 0.2162 | Classification: 0.2376 | Classification_binary: 0.2946 | HitGrouping: 0.2887 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4104 | Regression_eta: 0.0044 | Regression_pt: 0.4129 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1356 | Classification_binary: 0.2789 | Regression_cos_phi: 0.1560 | Regression_energy: 0.1819 | Regression_eta: 0.1217 | Regression_pt: 0.1920 | Regression_sin_phi: 0.1024 | LR: 9.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3391s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8669 | max_loss: 0.4301 | max_backward: 1.6617 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.9696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6691.5 | uss_MB: 5706.5 | child_rss_MB: 5415.0 | allocated_MB: 341.6 | reserved_MB: 23952.0 | inactive_split_MB: 277.1 +rank7 [2026-10-28 14:46:36,780] INFO : Step 5200/50000 rank7 | Train Loss: 2.5281 | Train Loss Unweighted: 1.4845 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2151 | Classification: 0.2850 | Classification_binary: 0.3669 | HitGrouping: 0.3041 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4080 | Regression_eta: 0.0050 | Regression_pt: 0.4142 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1805 | Classification_binary: 0.3685 | Regression_cos_phi: 0.2866 | Regression_energy: 0.1914 | Regression_eta: 0.1892 | Regression_pt: 0.2002 | Regression_sin_phi: 0.2602 | LR: 9.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9480s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0145 | max_forward: 0.9023 | max_loss: 0.4519 | max_backward: 1.3665 | max_optimizer_step: 0.1749 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5753.4 | uss_MB: 4770.0 | child_rss_MB: 5415.7 | allocated_MB: 344.4 | reserved_MB: 23952.0 | inactive_split_MB: 195.7 +rank7 [2026-10-28 14:50:33,442] INFO : Step 5300/50000 rank7 | Train Loss: 2.5101 | Train Loss Unweighted: 1.4783 | Train Loss Components: Auxiliary: 0.4843 | Cardinality: 0.2407 | Classification: 0.2577 | Classification_binary: 0.3775 | HitGrouping: 0.2944 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4132 | Regression_eta: 0.0046 | Regression_pt: 0.4201 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1234 | Classification_binary: 0.3388 | Regression_cos_phi: 0.1639 | Regression_energy: 0.1943 | Regression_eta: 0.1475 | Regression_pt: 0.2031 | Regression_sin_phi: 0.1414 | LR: 9.75e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.1868s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0065 | max_forward: 0.8286 | max_loss: 0.1827 | max_backward: 1.8076 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5755.2 | uss_MB: 4767.3 | child_rss_MB: 5415.6 | allocated_MB: 345.9 | reserved_MB: 25966.0 | inactive_split_MB: 190.2 +rank7 [2026-10-28 14:54:36,492] INFO : Step 5400/50000 rank7 | Train Loss: 2.3350 | Train Loss Unweighted: 1.2951 | Train Loss Components: Auxiliary: 0.4374 | Cardinality: 0.2481 | Classification: 0.2205 | Classification_binary: 0.3200 | HitGrouping: 0.2816 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3714 | Regression_eta: 0.0045 | Regression_pt: 0.3735 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.3466 | Regression_cos_phi: 0.2550 | Regression_energy: 0.1844 | Regression_eta: 0.1890 | Regression_pt: 0.1910 | Regression_sin_phi: 0.3155 | LR: 9.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4924s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.6400 | max_loss: 0.1445 | max_backward: 1.3062 | max_optimizer_step: 0.1742 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5750.9 | uss_MB: 4771.3 | child_rss_MB: 5391.8 | allocated_MB: 341.4 | reserved_MB: 25966.0 | inactive_split_MB: 237.1 +rank7 [2026-10-28 14:58:42,458] INFO : Step 5500/50000 rank7 | Train Loss: 2.4288 | Train Loss Unweighted: 1.4365 | Train Loss Components: Auxiliary: 0.4704 | Cardinality: 0.2092 | Classification: 0.2587 | Classification_binary: 0.3304 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4158 | Regression_eta: 0.0047 | Regression_pt: 0.4216 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1622 | Classification_binary: 0.3326 | Regression_cos_phi: 0.2112 | Regression_energy: 0.2075 | Regression_eta: 0.1880 | Regression_pt: 0.2180 | Regression_sin_phi: 0.2354 | LR: 9.73e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5908s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6516 | max_loss: 0.2472 | max_backward: 1.3949 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5747.6 | uss_MB: 4770.4 | child_rss_MB: 5393.4 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 191.3 +rank7 [2026-10-28 15:02:44,093] INFO : Step 5600/50000 rank7 | Train Loss: 2.7523 | Train Loss Unweighted: 1.5318 | Train Loss Components: Auxiliary: 0.5133 | Cardinality: 0.3067 | Classification: 0.3044 | Classification_binary: 0.4138 | HitGrouping: 0.2870 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3983 | Regression_eta: 0.0058 | Regression_pt: 0.4034 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1951 | Classification_binary: 0.4185 | Regression_cos_phi: 0.2839 | Regression_energy: 0.1933 | Regression_eta: 0.1996 | Regression_pt: 0.2035 | Regression_sin_phi: 0.2530 | LR: 9.72e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6393s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.9715 | max_loss: 0.2254 | max_backward: 2.2912 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5750.8 | uss_MB: 4761.9 | child_rss_MB: 5391.2 | allocated_MB: 345.3 | reserved_MB: 25966.0 | inactive_split_MB: 273.7 +rank7 [2026-10-28 15:06:32,821] INFO : Step 5700/50000 rank7 | Train Loss: 2.2632 | Train Loss Unweighted: 1.3603 | Train Loss Components: Auxiliary: 0.4447 | Cardinality: 0.1906 | Classification: 0.2182 | Classification_binary: 0.3151 | HitGrouping: 0.2769 | Regression_cos_phi: 0.0021 | Regression_energy: 0.4051 | Regression_eta: 0.0050 | Regression_pt: 0.4126 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1531 | Classification_binary: 0.3570 | Regression_cos_phi: 0.2205 | Regression_energy: 0.2243 | Regression_eta: 0.1777 | Regression_pt: 0.2376 | Regression_sin_phi: 0.2049 | LR: 9.71e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.1505s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.8386 | max_loss: 0.1870 | max_backward: 1.6807 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.6026 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 5751.4 | uss_MB: 4773.7 | child_rss_MB: 5393.7 | allocated_MB: 343.4 | reserved_MB: 25966.0 | inactive_split_MB: 277.2 +rank7 [2026-10-28 15:10:43,467] INFO : Step 5800/50000 rank7 | Train Loss: 2.3369 | Train Loss Unweighted: 1.3552 | Train Loss Components: Auxiliary: 0.4487 | Cardinality: 0.2219 | Classification: 0.2402 | Classification_binary: 0.3025 | HitGrouping: 0.2755 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4014 | Regression_eta: 0.0045 | Regression_pt: 0.4014 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1323 | Classification_binary: 0.3088 | Regression_cos_phi: 0.2046 | Regression_energy: 0.2108 | Regression_eta: 0.1275 | Regression_pt: 0.2186 | Regression_sin_phi: 0.1509 | LR: 9.70e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5294s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0068 | max_forward: 0.6770 | max_loss: 0.2353 | max_backward: 1.2083 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.6551 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5749.0 | uss_MB: 4775.9 | child_rss_MB: 5376.8 | allocated_MB: 342.5 | reserved_MB: 25966.0 | inactive_split_MB: 183.7 +rank7 [2026-10-28 15:14:48,144] INFO : Step 5900/50000 rank7 | Train Loss: 2.6448 | Train Loss Unweighted: 1.4868 | Train Loss Components: Auxiliary: 0.4912 | Cardinality: 0.2536 | Classification: 0.2773 | Classification_binary: 0.3805 | HitGrouping: 0.2796 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4060 | Regression_eta: 0.0060 | Regression_pt: 0.4110 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1840 | Classification_binary: 0.3783 | Regression_cos_phi: 0.2539 | Regression_energy: 0.1963 | Regression_eta: 0.2425 | Regression_pt: 0.2066 | Regression_sin_phi: 0.3151 | LR: 9.69e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.0721s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 1.2087 | max_loss: 0.3503 | max_backward: 2.7435 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6037 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5750.2 | uss_MB: 4778.8 | child_rss_MB: 5371.2 | allocated_MB: 345.5 | reserved_MB: 25966.0 | inactive_split_MB: 206.0 +rank7 [2026-10-28 15:18:45,594] INFO : Step 6000/50000 rank7 | Train Loss: 2.4633 | Train Loss Unweighted: 1.4115 | Train Loss Components: Auxiliary: 0.4815 | Cardinality: 0.2725 | Classification: 0.2579 | Classification_binary: 0.3590 | HitGrouping: 0.2771 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3919 | Regression_eta: 0.0047 | Regression_pt: 0.3931 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1310 | Classification_binary: 0.3716 | Regression_cos_phi: 0.1716 | Regression_energy: 0.1689 | Regression_eta: 0.1782 | Regression_pt: 0.1765 | Regression_sin_phi: 0.1725 | LR: 9.68e-04 | DataLoad Time: 0.0098s | Model Forward Time: 6.3901s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0224 | max_forward: 1.7138 | max_loss: 0.2608 | max_backward: 2.5718 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5258 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5754.0 | uss_MB: 4530.5 | child_rss_MB: 5366.5 | allocated_MB: 341.6 | reserved_MB: 25966.0 | inactive_split_MB: 424.6 +rank7 [2026-10-28 15:18:46,644] INFO : Running validation on rank7 +rank7 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 15:18:46,644] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 15:19:34,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:19:35,273] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 15:19:56,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 15:19:57,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 15:19:59,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 15:20:00,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 15:20:10,406] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-28 15:20:11,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:20:11,686] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 15:20:27,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 15:20:28,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 15:20:29,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 15:20:30,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 15:20:40,533] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 15:20:40,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:20:41,486] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 15:20:57,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 15:20:57,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 15:20:58,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 15:20:58,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_6000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 15:21:09,002] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 15:25:37,243] INFO : Step 6100/50000 rank7 | Train Loss: 2.0900 | Train Loss Unweighted: 1.2144 | Train Loss Components: Auxiliary: 0.4098 | Cardinality: 0.1965 | Classification: 0.1988 | Classification_binary: 0.2682 | HitGrouping: 0.2730 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3698 | Regression_eta: 0.0043 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.2407 | Regression_cos_phi: 0.1139 | Regression_energy: 0.1638 | Regression_eta: 0.1001 | Regression_pt: 0.1692 | Regression_sin_phi: 0.1611 | LR: 9.67e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2788s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6449 | max_loss: 0.3961 | max_backward: 1.3506 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 1.2117 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7180.9 | uss_MB: 6201.5 | child_rss_MB: 5361.2 | allocated_MB: 341.8 | reserved_MB: 24856.0 | inactive_split_MB: 272.9 +rank7 [2026-10-28 15:29:27,137] INFO : Step 6200/50000 rank7 | Train Loss: 2.5044 | Train Loss Unweighted: 1.3941 | Train Loss Components: Auxiliary: 0.4763 | Cardinality: 0.3244 | Classification: 0.2542 | Classification_binary: 0.3177 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4019 | Regression_eta: 0.0049 | Regression_pt: 0.4103 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1482 | Classification_binary: 0.3511 | Regression_cos_phi: 0.1884 | Regression_energy: 0.1979 | Regression_eta: 0.1576 | Regression_pt: 0.2089 | Regression_sin_phi: 0.2282 | LR: 9.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.3978s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6093 | max_loss: 0.1402 | max_backward: 1.3174 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.5349 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5334.3 | allocated_MB: 342.9 | reserved_MB: 25886.0 | inactive_split_MB: 275.3 +rank7 [2026-10-28 15:33:16,837] INFO : Step 6300/50000 rank7 | Train Loss: 2.5286 | Train Loss Unweighted: 1.4468 | Train Loss Components: Auxiliary: 0.4879 | Cardinality: 0.2769 | Classification: 0.2744 | Classification_binary: 0.3851 | HitGrouping: 0.2778 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3852 | Regression_eta: 0.0049 | Regression_pt: 0.3922 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3991 | Regression_cos_phi: 0.1962 | Regression_energy: 0.1920 | Regression_eta: 0.1475 | Regression_pt: 0.2033 | Regression_sin_phi: 0.1852 | LR: 9.65e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2457s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7197 | max_loss: 0.2287 | max_backward: 1.5551 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.9389 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5863.0 | uss_MB: 4888.6 | child_rss_MB: 5359.0 | allocated_MB: 345.8 | reserved_MB: 26864.0 | inactive_split_MB: 446.8 +rank7 [2026-10-28 15:37:09,470] INFO : Step 6400/50000 rank7 | Train Loss: 2.2226 | Train Loss Unweighted: 1.2894 | Train Loss Components: Auxiliary: 0.4304 | Cardinality: 0.2378 | Classification: 0.2160 | Classification_binary: 0.3068 | HitGrouping: 0.2717 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3771 | Regression_eta: 0.0043 | Regression_pt: 0.3809 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1385 | Classification_binary: 0.3585 | Regression_cos_phi: 0.1722 | Regression_energy: 0.1661 | Regression_eta: 0.1849 | Regression_pt: 0.1708 | Regression_sin_phi: 0.1969 | LR: 9.64e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.9970s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8669 | max_loss: 0.2262 | max_backward: 1.3701 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.8537 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1233 | Memory: rss_MB: 5862.8 | uss_MB: 4886.2 | child_rss_MB: 5357.6 | allocated_MB: 338.5 | reserved_MB: 27896.0 | inactive_split_MB: 268.7 +rank7 [2026-10-28 15:41:11,634] INFO : Step 6500/50000 rank7 | Train Loss: 2.1576 | Train Loss Unweighted: 1.2524 | Train Loss Components: Auxiliary: 0.4147 | Cardinality: 0.2002 | Classification: 0.2161 | Classification_binary: 0.2978 | HitGrouping: 0.2705 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3614 | Regression_eta: 0.0044 | Regression_pt: 0.3683 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1408 | Classification_binary: 0.3056 | Regression_cos_phi: 0.2252 | Regression_energy: 0.1875 | Regression_eta: 0.1903 | Regression_pt: 0.1953 | Regression_sin_phi: 0.1494 | LR: 9.63e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6724s | Phase Times: max_pre_step_barrier: 0.0165 | max_batch_to_device: 0.0152 | max_forward: 1.3911 | max_loss: 0.3513 | max_backward: 2.0112 | max_optimizer_step: 0.1776 | max_tensorboard_logging: 0.8121 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5861.0 | uss_MB: 4889.6 | child_rss_MB: 5357.0 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 411.0 +rank7 [2026-10-28 15:45:09,041] INFO : Step 6600/50000 rank7 | Train Loss: 2.5107 | Train Loss Unweighted: 1.4389 | Train Loss Components: Auxiliary: 0.4838 | Cardinality: 0.2696 | Classification: 0.2507 | Classification_binary: 0.3848 | HitGrouping: 0.2704 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3953 | Regression_eta: 0.0048 | Regression_pt: 0.3981 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.3659 | Regression_cos_phi: 0.1705 | Regression_energy: 0.2069 | Regression_eta: 0.1632 | Regression_pt: 0.2183 | Regression_sin_phi: 0.1696 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2532s | Phase Times: max_pre_step_barrier: 0.0672 | max_batch_to_device: 0.0828 | max_forward: 0.8302 | max_loss: 0.1594 | max_backward: 1.7791 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5881 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5863.8 | uss_MB: 4892.6 | child_rss_MB: 5324.1 | allocated_MB: 343.8 | reserved_MB: 27896.0 | inactive_split_MB: 260.2 +rank7 [2026-10-28 15:49:06,066] INFO : Step 6700/50000 rank7 | Train Loss: 2.2341 | Train Loss Unweighted: 1.2731 | Train Loss Components: Auxiliary: 0.4297 | Cardinality: 0.2367 | Classification: 0.2163 | Classification_binary: 0.3171 | HitGrouping: 0.2677 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3643 | Regression_eta: 0.0044 | Regression_pt: 0.3664 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1159 | Classification_binary: 0.2781 | Regression_cos_phi: 0.1573 | Regression_energy: 0.1701 | Regression_eta: 0.1630 | Regression_pt: 0.1745 | Regression_sin_phi: 0.1611 | LR: 9.61e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.0300s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8578 | max_loss: 0.1773 | max_backward: 1.7349 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0784 | Memory: rss_MB: 5864.7 | uss_MB: 4892.9 | child_rss_MB: 5346.8 | allocated_MB: 344.8 | reserved_MB: 27896.0 | inactive_split_MB: 187.2 +rank7 [2026-10-28 15:52:50,801] INFO : Step 6800/50000 rank7 | Train Loss: 2.1405 | Train Loss Unweighted: 1.2407 | Train Loss Components: Auxiliary: 0.4140 | Cardinality: 0.1926 | Classification: 0.2195 | Classification_binary: 0.2879 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3615 | Regression_eta: 0.0044 | Regression_pt: 0.3630 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1490 | Classification_binary: 0.3171 | Regression_cos_phi: 0.1945 | Regression_energy: 0.1889 | Regression_eta: 0.1518 | Regression_pt: 0.1979 | Regression_sin_phi: 0.2002 | LR: 9.60e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8870s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.6032 | max_loss: 0.1456 | max_backward: 1.1738 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 1.0666 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5858.1 | uss_MB: 4891.3 | child_rss_MB: 5347.4 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 228.0 +rank7 [2026-10-28 15:56:38,729] INFO : Step 6900/50000 rank7 | Train Loss: 2.3865 | Train Loss Unweighted: 1.3618 | Train Loss Components: Auxiliary: 0.4537 | Cardinality: 0.2286 | Classification: 0.2666 | Classification_binary: 0.3427 | HitGrouping: 0.2688 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3681 | Regression_eta: 0.0049 | Regression_pt: 0.3742 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1657 | Classification_binary: 0.3751 | Regression_cos_phi: 0.2003 | Regression_energy: 0.1763 | Regression_eta: 0.1737 | Regression_pt: 0.1856 | Regression_sin_phi: 0.2330 | LR: 9.58e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.8827s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.6680 | max_loss: 0.1505 | max_backward: 1.2324 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.9217 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5845.9 | uss_MB: 4896.5 | child_rss_MB: 5327.8 | allocated_MB: 343.5 | reserved_MB: 27896.0 | inactive_split_MB: 175.6 +rank7 [2026-10-28 16:00:37,729] INFO : Step 7000/50000 rank7 | Train Loss: 2.5407 | Train Loss Unweighted: 1.4018 | Train Loss Components: Auxiliary: 0.4695 | Cardinality: 0.2641 | Classification: 0.2490 | Classification_binary: 0.3754 | HitGrouping: 0.2737 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3819 | Regression_eta: 0.0051 | Regression_pt: 0.3842 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1441 | Classification_binary: 0.4051 | Regression_cos_phi: 0.2843 | Regression_energy: 0.1899 | Regression_eta: 0.1843 | Regression_pt: 0.1987 | Regression_sin_phi: 0.2162 | LR: 9.57e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4737s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9061 | max_loss: 0.1557 | max_backward: 1.8187 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8492 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5835.4 | uss_MB: 4899.5 | child_rss_MB: 5356.5 | allocated_MB: 345.0 | reserved_MB: 27896.0 | inactive_split_MB: 271.6 +rank7 [2026-10-28 16:00:38,672] INFO : Running validation on rank7 +rank7 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:00:38,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:01:32,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:01:33,505] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 16:02:00,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:02:01,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:02:02,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:02:03,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:02:13,026] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 16:02:13,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:02:14,069] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 16:02:30,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:02:31,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:02:32,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:02:33,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:02:43,680] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:02:43,760] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:02:44,278] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 16:02:59,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:03:00,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:03:00,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:03:01,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_7000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:03:11,514] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 16:07:39,193] INFO : Step 7100/50000 rank7 | Train Loss: 2.0262 | Train Loss Unweighted: 1.1614 | Train Loss Components: Auxiliary: 0.3878 | Cardinality: 0.1739 | Classification: 0.1885 | Classification_binary: 0.2767 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3424 | Regression_eta: 0.0040 | Regression_pt: 0.3454 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2962 | Regression_cos_phi: 0.1456 | Regression_energy: 0.1725 | Regression_eta: 0.1444 | Regression_pt: 0.1807 | Regression_sin_phi: 0.1329 | LR: 9.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5597s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.6584 | max_loss: 0.2486 | max_backward: 1.6284 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 1.1784 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 7494.3 | uss_MB: 6550.8 | child_rss_MB: 5354.7 | allocated_MB: 342.6 | reserved_MB: 23102.0 | inactive_split_MB: 252.5 +rank7 [2026-10-28 16:11:41,310] INFO : Step 7200/50000 rank7 | Train Loss: 2.3960 | Train Loss Unweighted: 1.3173 | Train Loss Components: Auxiliary: 0.4586 | Cardinality: 0.2893 | Classification: 0.2420 | Classification_binary: 0.3389 | HitGrouping: 0.2672 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3624 | Regression_eta: 0.0045 | Regression_pt: 0.3645 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1483 | Classification_binary: 0.3783 | Regression_cos_phi: 0.1493 | Regression_energy: 0.2031 | Regression_eta: 0.1573 | Regression_pt: 0.2083 | Regression_sin_phi: 0.1925 | LR: 9.55e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9566s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7483 | max_loss: 0.1628 | max_backward: 1.4263 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 5916.7 | uss_MB: 4980.4 | child_rss_MB: 5360.7 | allocated_MB: 347.7 | reserved_MB: 23818.0 | inactive_split_MB: 502.9 +rank7 [2026-10-28 16:15:45,431] INFO : Step 7300/50000 rank7 | Train Loss: 2.3562 | Train Loss Unweighted: 1.2926 | Train Loss Components: Auxiliary: 0.4457 | Cardinality: 0.2504 | Classification: 0.2282 | Classification_binary: 0.3623 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3460 | Regression_eta: 0.0046 | Regression_pt: 0.3463 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1282 | Classification_binary: 0.3592 | Regression_cos_phi: 0.1960 | Regression_energy: 0.1685 | Regression_eta: 0.1417 | Regression_pt: 0.1740 | Regression_sin_phi: 0.1839 | LR: 9.53e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.9015s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0068 | max_forward: 0.7296 | max_loss: 0.4336 | max_backward: 1.6404 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 5898.8 | uss_MB: 4979.8 | child_rss_MB: 5356.1 | allocated_MB: 345.1 | reserved_MB: 23818.0 | inactive_split_MB: 424.0 +rank7 [2026-10-28 16:19:47,790] INFO : Step 7400/50000 rank7 | Train Loss: 2.0534 | Train Loss Unweighted: 1.2012 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.1939 | Classification: 0.1693 | Classification_binary: 0.2886 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3668 | Regression_eta: 0.0036 | Regression_pt: 0.3687 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2544 | Regression_cos_phi: 0.1328 | Regression_energy: 0.1622 | Regression_eta: 0.0982 | Regression_pt: 0.1662 | Regression_sin_phi: 0.1450 | LR: 9.52e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6027s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0355 | max_forward: 0.7008 | max_loss: 0.1606 | max_backward: 1.4394 | max_optimizer_step: 0.1802 | max_tensorboard_logging: 0.4863 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5890.8 | uss_MB: 4980.8 | child_rss_MB: 5359.8 | allocated_MB: 340.3 | reserved_MB: 23818.0 | inactive_split_MB: 179.7 +rank7 [2026-10-28 16:23:56,337] INFO : Step 7500/50000 rank7 | Train Loss: 2.0783 | Train Loss Unweighted: 1.1891 | Train Loss Components: Auxiliary: 0.4032 | Cardinality: 0.1917 | Classification: 0.1930 | Classification_binary: 0.2821 | HitGrouping: 0.2622 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3508 | Regression_eta: 0.0040 | Regression_pt: 0.3547 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3049 | Regression_cos_phi: 0.1885 | Regression_energy: 0.1990 | Regression_eta: 0.1728 | Regression_pt: 0.2073 | Regression_sin_phi: 0.1826 | LR: 9.51e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7916s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6261 | max_loss: 0.2479 | max_backward: 1.3106 | max_optimizer_step: 0.2055 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5892.9 | uss_MB: 4987.1 | child_rss_MB: 5342.4 | allocated_MB: 343.5 | reserved_MB: 24124.0 | inactive_split_MB: 200.5 +rank7 [2026-10-28 16:27:58,400] INFO : Step 7600/50000 rank7 | Train Loss: 2.5549 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4826 | Cardinality: 0.3624 | Classification: 0.2640 | Classification_binary: 0.3657 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3733 | Regression_eta: 0.0049 | Regression_pt: 0.3779 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.3881 | Regression_cos_phi: 0.1655 | Regression_energy: 0.2017 | Regression_eta: 0.1796 | Regression_pt: 0.2131 | Regression_sin_phi: 0.1748 | LR: 9.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7002s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.8483 | max_loss: 0.1481 | max_backward: 1.6540 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 1.1276 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5887.4 | uss_MB: 4984.6 | child_rss_MB: 5352.8 | allocated_MB: 347.2 | reserved_MB: 24124.0 | inactive_split_MB: 207.6 +rank7 [2026-10-28 16:32:00,293] INFO : Step 7700/50000 rank7 | Train Loss: 2.2823 | Train Loss Unweighted: 1.1775 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.3508 | Classification: 0.1881 | Classification_binary: 0.3342 | HitGrouping: 0.2597 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3222 | Regression_eta: 0.0042 | Regression_pt: 0.3243 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.3378 | Regression_cos_phi: 0.1162 | Regression_energy: 0.1593 | Regression_eta: 0.1416 | Regression_pt: 0.1639 | Regression_sin_phi: 0.1696 | LR: 9.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9800s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 1.0795 | max_loss: 0.2028 | max_backward: 2.1528 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.9289 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5886.4 | uss_MB: 4990.1 | child_rss_MB: 5357.9 | allocated_MB: 343.0 | reserved_MB: 24124.0 | inactive_split_MB: 190.6 +rank7 [2026-10-28 16:35:55,393] INFO : Step 7800/50000 rank7 | Train Loss: 2.0317 | Train Loss Unweighted: 1.1606 | Train Loss Components: Auxiliary: 0.3922 | Cardinality: 0.1977 | Classification: 0.2024 | Classification_binary: 0.2672 | HitGrouping: 0.2664 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3405 | Regression_eta: 0.0039 | Regression_pt: 0.3424 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1254 | Classification_binary: 0.2715 | Regression_cos_phi: 0.1801 | Regression_energy: 0.1608 | Regression_eta: 0.1466 | Regression_pt: 0.1658 | Regression_sin_phi: 0.1837 | LR: 9.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2111s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.5886 | max_loss: 0.1643 | max_backward: 1.5565 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 1.0039 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0338 | Memory: rss_MB: 5873.5 | uss_MB: 4989.6 | child_rss_MB: 5367.2 | allocated_MB: 340.1 | reserved_MB: 24124.0 | inactive_split_MB: 205.1 +rank7 [2026-10-28 16:40:11,802] INFO : Step 7900/50000 rank7 | Train Loss: 2.2870 | Train Loss Unweighted: 1.3038 | Train Loss Components: Auxiliary: 0.4367 | Cardinality: 0.2124 | Classification: 0.2456 | Classification_binary: 0.3403 | HitGrouping: 0.2659 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3527 | Regression_eta: 0.0045 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1819 | Classification_binary: 0.3384 | Regression_cos_phi: 0.2690 | Regression_energy: 0.1743 | Regression_eta: 0.1700 | Regression_pt: 0.1810 | Regression_sin_phi: 0.1990 | LR: 9.46e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3417s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0803 | max_forward: 0.8348 | max_loss: 0.1790 | max_backward: 1.5908 | max_optimizer_step: 0.2647 | max_tensorboard_logging: 0.8976 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5856.2 | uss_MB: 4994.7 | child_rss_MB: 5365.7 | allocated_MB: 344.0 | reserved_MB: 24124.0 | inactive_split_MB: 268.0 +rank7 [2026-10-28 16:44:12,790] INFO : Step 8000/50000 rank7 | Train Loss: 2.2983 | Train Loss Unweighted: 1.2849 | Train Loss Components: Auxiliary: 0.4436 | Cardinality: 0.2443 | Classification: 0.2335 | Classification_binary: 0.3511 | HitGrouping: 0.2652 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3432 | Regression_eta: 0.0045 | Regression_pt: 0.3479 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1283 | Classification_binary: 0.3492 | Regression_cos_phi: 0.1577 | Regression_energy: 0.1677 | Regression_eta: 0.1452 | Regression_pt: 0.1767 | Regression_sin_phi: 0.1366 | LR: 9.44e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2445s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.9359 | max_loss: 0.2587 | max_backward: 1.8554 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5063 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5853.3 | uss_MB: 4996.6 | child_rss_MB: 5364.4 | allocated_MB: 342.6 | reserved_MB: 24124.0 | inactive_split_MB: 182.9 +rank7 [2026-10-28 16:44:13,936] INFO : Running validation on rank7 +rank7 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:44:13,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:45:04,006] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:45:04,567] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 16:45:30,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:45:31,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:45:32,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:45:34,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:45:44,402] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 16:45:44,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:45:45,473] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 16:46:01,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:46:02,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:46:03,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:46:04,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:46:14,181] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:46:14,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:46:15,125] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 16:46:30,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:46:31,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:46:31,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:46:32,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_8000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:46:42,459] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 16:51:07,904] INFO : Step 8100/50000 rank7 | Train Loss: 1.9014 | Train Loss Unweighted: 1.1111 | Train Loss Components: Auxiliary: 0.3801 | Cardinality: 0.1804 | Classification: 0.1733 | Classification_binary: 0.2742 | HitGrouping: 0.2557 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3275 | Regression_eta: 0.0033 | Regression_pt: 0.3292 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1476 | Classification_binary: 0.3021 | Regression_cos_phi: 0.1767 | Regression_energy: 0.1659 | Regression_eta: 0.1441 | Regression_pt: 0.1745 | Regression_sin_phi: 0.1549 | LR: 9.43e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7756s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7388 | max_loss: 0.4359 | max_backward: 1.8404 | max_optimizer_step: 0.1797 | max_tensorboard_logging: 0.9067 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7581.6 | uss_MB: 5901.8 | child_rss_MB: 5369.6 | allocated_MB: 341.2 | reserved_MB: 22616.0 | inactive_split_MB: 269.0 +rank7 [2026-10-28 16:55:30,392] INFO : Step 8200/50000 rank7 | Train Loss: 2.1575 | Train Loss Unweighted: 1.1695 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2436 | Classification: 0.2229 | Classification_binary: 0.3015 | HitGrouping: 0.2572 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3156 | Regression_eta: 0.0044 | Regression_pt: 0.3204 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1445 | Classification_binary: 0.3494 | Regression_cos_phi: 0.2140 | Regression_energy: 0.1785 | Regression_eta: 0.1760 | Regression_pt: 0.1885 | Regression_sin_phi: 0.1755 | LR: 9.42e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1550s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0276 | max_forward: 0.9549 | max_loss: 0.4225 | max_backward: 1.5890 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.8901 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5945.7 | uss_MB: 5088.1 | child_rss_MB: 5361.5 | allocated_MB: 343.9 | reserved_MB: 23518.0 | inactive_split_MB: 243.9 +rank7 [2026-10-28 16:59:31,718] INFO : Step 8300/50000 rank7 | Train Loss: 2.3259 | Train Loss Unweighted: 1.2765 | Train Loss Components: Auxiliary: 0.4423 | Cardinality: 0.2469 | Classification: 0.2228 | Classification_binary: 0.3662 | HitGrouping: 0.2625 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3365 | Regression_eta: 0.0052 | Regression_pt: 0.3410 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1267 | Classification_binary: 0.3740 | Regression_cos_phi: 0.1779 | Regression_energy: 0.1711 | Regression_eta: 0.1914 | Regression_pt: 0.1772 | Regression_sin_phi: 0.1340 | LR: 9.40e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.9202s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7498 | max_loss: 0.2504 | max_backward: 3.0087 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.2764 | Memory: rss_MB: 5947.9 | uss_MB: 5090.0 | child_rss_MB: 5358.0 | allocated_MB: 345.9 | reserved_MB: 23794.0 | inactive_split_MB: 250.9 +rank7 [2026-10-28 17:03:26,315] INFO : Step 8400/50000 rank7 | Train Loss: 2.0372 | Train Loss Unweighted: 1.1436 | Train Loss Components: Auxiliary: 0.3860 | Cardinality: 0.1960 | Classification: 0.1954 | Classification_binary: 0.2923 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3235 | Regression_eta: 0.0040 | Regression_pt: 0.3240 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0990 | Classification_binary: 0.3006 | Regression_cos_phi: 0.1690 | Regression_energy: 0.1629 | Regression_eta: 0.1323 | Regression_pt: 0.1676 | Regression_sin_phi: 0.1631 | LR: 9.39e-04 | DataLoad Time: 0.0234s | Model Forward Time: 6.0439s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.9015 | max_loss: 0.2444 | max_backward: 1.4494 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 2.8975 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5944.2 | uss_MB: 5085.8 | child_rss_MB: 5356.9 | allocated_MB: 341.7 | reserved_MB: 23794.0 | inactive_split_MB: 198.5 +rank7 [2026-10-28 17:07:18,317] INFO : Step 8500/50000 rank7 | Train Loss: 1.9112 | Train Loss Unweighted: 1.0833 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.1776 | Classification: 0.1893 | Classification_binary: 0.2624 | HitGrouping: 0.2535 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3111 | Regression_eta: 0.0038 | Regression_pt: 0.3129 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.2480 | Regression_cos_phi: 0.1805 | Regression_energy: 0.1574 | Regression_eta: 0.1313 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1399 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4973s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8691 | max_loss: 0.2076 | max_backward: 1.8132 | max_optimizer_step: 0.2972 | max_tensorboard_logging: 0.8571 | max_tensorboard_flush: 0.0068 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5941.4 | uss_MB: 4990.5 | child_rss_MB: 5326.6 | allocated_MB: 344.3 | reserved_MB: 24540.0 | inactive_split_MB: 207.9 +rank7 [2026-10-28 17:11:31,152] INFO : Step 8600/50000 rank7 | Train Loss: 2.1330 | Train Loss Unweighted: 1.2197 | Train Loss Components: Auxiliary: 0.4138 | Cardinality: 0.2169 | Classification: 0.2156 | Classification_binary: 0.3300 | HitGrouping: 0.2516 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3325 | Regression_eta: 0.0041 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.3436 | Regression_cos_phi: 0.1253 | Regression_energy: 0.1502 | Regression_eta: 0.1209 | Regression_pt: 0.1543 | Regression_sin_phi: 0.1819 | LR: 9.36e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1709s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7354 | max_loss: 0.1680 | max_backward: 1.8639 | max_optimizer_step: 0.1634 | max_tensorboard_logging: 0.5576 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5942.2 | uss_MB: 4996.0 | child_rss_MB: 5330.5 | allocated_MB: 346.0 | reserved_MB: 25564.0 | inactive_split_MB: 262.8 +rank7 [2026-10-28 17:15:44,852] INFO : Step 8700/50000 rank7 | Train Loss: 2.0620 | Train Loss Unweighted: 1.1443 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2240 | Classification: 0.1879 | Classification_binary: 0.3223 | HitGrouping: 0.2544 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3134 | Regression_eta: 0.0039 | Regression_pt: 0.3127 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0965 | Classification_binary: 0.3187 | Regression_cos_phi: 0.1649 | Regression_energy: 0.1398 | Regression_eta: 0.1372 | Regression_pt: 0.1442 | Regression_sin_phi: 0.1540 | LR: 9.34e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2148s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0158 | max_forward: 1.0476 | max_loss: 0.2014 | max_backward: 1.7329 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5940.8 | uss_MB: 5094.4 | child_rss_MB: 5330.4 | allocated_MB: 342.2 | reserved_MB: 25564.0 | inactive_split_MB: 216.7 +rank7 [2026-10-28 17:19:40,254] INFO : Step 8800/50000 rank7 | Train Loss: 1.7949 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.1759 | Classification: 0.1706 | Classification_binary: 0.2454 | HitGrouping: 0.2496 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2965 | Regression_eta: 0.0032 | Regression_pt: 0.2960 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1074 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1229 | Regression_energy: 0.1371 | Regression_eta: 0.0801 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0964 | LR: 9.33e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.4425s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6065 | max_loss: 0.1648 | max_backward: 1.1598 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.4931 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5941.3 | uss_MB: 4934.1 | child_rss_MB: 5319.4 | allocated_MB: 343.9 | reserved_MB: 25564.0 | inactive_split_MB: 214.4 +rank7 [2026-10-28 17:23:38,392] INFO : Step 8900/50000 rank7 | Train Loss: 2.1886 | Train Loss Unweighted: 1.2309 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2154 | Classification: 0.2111 | Classification_binary: 0.3257 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3413 | Regression_eta: 0.0043 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3099 | Regression_cos_phi: 0.1759 | Regression_energy: 0.1648 | Regression_eta: 0.1356 | Regression_pt: 0.1713 | Regression_sin_phi: 0.1532 | LR: 9.31e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6543s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.2436 | max_backward: 1.4220 | max_optimizer_step: 0.1822 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 5934.1 | uss_MB: 5099.4 | child_rss_MB: 5328.0 | allocated_MB: 344.4 | reserved_MB: 25564.0 | inactive_split_MB: 191.6 +rank7 [2026-10-28 17:27:33,261] INFO : Step 9000/50000 rank7 | Train Loss: 2.1387 | Train Loss Unweighted: 1.2070 | Train Loss Components: Auxiliary: 0.4104 | Cardinality: 0.2160 | Classification: 0.2046 | Classification_binary: 0.3464 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3226 | Regression_eta: 0.0041 | Regression_pt: 0.3250 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.3438 | Regression_cos_phi: 0.2195 | Regression_energy: 0.1634 | Regression_eta: 0.1445 | Regression_pt: 0.1711 | Regression_sin_phi: 0.2097 | LR: 9.30e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.9355 | max_loss: 0.2134 | max_backward: 2.3071 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5408 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 5934.5 | uss_MB: 5005.1 | child_rss_MB: 5334.6 | allocated_MB: 343.7 | reserved_MB: 25564.0 | inactive_split_MB: 280.2 +rank7 [2026-10-28 17:27:34,333] INFO : Running validation on rank7 +rank7 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 17:27:34,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 17:28:24,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:28:25,268] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 17:28:51,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 17:28:52,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 17:28:53,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 17:28:54,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 17:29:04,748] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 17:29:05,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:29:06,071] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 17:29:22,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 17:29:23,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 17:29:24,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 17:29:26,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 17:29:36,317] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 17:29:37,138] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:29:37,692] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 17:29:52,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 17:29:53,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 17:29:53,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 17:29:54,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_9000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 17:30:04,511] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 17:34:23,898] INFO : Step 9100/50000 rank7 | Train Loss: 1.7616 | Train Loss Unweighted: 1.0033 | Train Loss Components: Auxiliary: 0.3462 | Cardinality: 0.1678 | Classification: 0.1551 | Classification_binary: 0.2628 | HitGrouping: 0.2456 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2885 | Regression_eta: 0.0033 | Regression_pt: 0.2904 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0917 | Classification_binary: 0.2515 | Regression_cos_phi: 0.1385 | Regression_energy: 0.1441 | Regression_eta: 0.1321 | Regression_pt: 0.1482 | Regression_sin_phi: 0.1447 | LR: 9.28e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1045s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7079 | max_loss: 0.2436 | max_backward: 1.2774 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.8844 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7737.0 | uss_MB: 6899.4 | child_rss_MB: 5311.5 | allocated_MB: 340.8 | reserved_MB: 22710.0 | inactive_split_MB: 267.9 +rank7 [2026-10-28 17:38:24,814] INFO : Step 9200/50000 rank7 | Train Loss: 1.9843 | Train Loss Unweighted: 1.1063 | Train Loss Components: Auxiliary: 0.3779 | Cardinality: 0.2123 | Classification: 0.1931 | Classification_binary: 0.2846 | HitGrouping: 0.2528 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0038 | Regression_pt: 0.3123 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.3067 | Regression_cos_phi: 0.1480 | Regression_energy: 0.1485 | Regression_eta: 0.1330 | Regression_pt: 0.1537 | Regression_sin_phi: 0.2216 | LR: 9.27e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3254s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6707 | max_loss: 0.1446 | max_backward: 1.4400 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6012.1 | uss_MB: 5172.6 | child_rss_MB: 5328.6 | allocated_MB: 343.3 | reserved_MB: 24684.0 | inactive_split_MB: 193.2 +rank7 [2026-10-28 17:42:44,641] INFO : Step 9300/50000 rank7 | Train Loss: 2.2727 | Train Loss Unweighted: 1.2479 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2563 | Classification: 0.2147 | Classification_binary: 0.3456 | HitGrouping: 0.2638 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3348 | Regression_eta: 0.0045 | Regression_pt: 0.3437 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1285 | Classification_binary: 0.3482 | Regression_cos_phi: 0.2391 | Regression_energy: 0.1629 | Regression_eta: 0.1792 | Regression_pt: 0.1724 | Regression_sin_phi: 0.1742 | LR: 9.25e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.8269s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0191 | max_forward: 0.8505 | max_loss: 0.4666 | max_backward: 1.9718 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.8269 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6015.5 | uss_MB: 5178.2 | child_rss_MB: 5334.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 300.9 +rank7 [2026-10-28 17:46:50,464] INFO : Step 9400/50000 rank7 | Train Loss: 1.9266 | Train Loss Unweighted: 1.0619 | Train Loss Components: Auxiliary: 0.3687 | Cardinality: 0.1961 | Classification: 0.1720 | Classification_binary: 0.2822 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2993 | Regression_eta: 0.0038 | Regression_pt: 0.3007 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0961 | Classification_binary: 0.3052 | Regression_cos_phi: 0.1286 | Regression_energy: 0.1448 | Regression_eta: 0.0869 | Regression_pt: 0.1500 | Regression_sin_phi: 0.1334 | LR: 9.24e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4124s | Phase Times: max_pre_step_barrier: 0.0245 | max_batch_to_device: 0.0066 | max_forward: 0.7242 | max_loss: 0.4763 | max_backward: 1.6229 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.8787 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 6015.5 | uss_MB: 4422.3 | child_rss_MB: 5339.5 | allocated_MB: 343.0 | reserved_MB: 24684.0 | inactive_split_MB: 281.8 +rank7 [2026-10-28 17:51:11,694] INFO : Step 9500/50000 rank7 | Train Loss: 1.8426 | Train Loss Unweighted: 1.0252 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.1807 | Classification: 0.1606 | Classification_binary: 0.2447 | HitGrouping: 0.2522 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3059 | Regression_eta: 0.0036 | Regression_pt: 0.3066 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2197 | Regression_cos_phi: 0.1514 | Regression_energy: 0.1458 | Regression_eta: 0.1246 | Regression_pt: 0.1532 | Regression_sin_phi: 0.1011 | LR: 9.22e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.2521s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5823 | max_loss: 0.1521 | max_backward: 1.1256 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4719 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0304 | Memory: rss_MB: 6014.4 | uss_MB: 5178.6 | child_rss_MB: 5330.7 | allocated_MB: 343.5 | reserved_MB: 24684.0 | inactive_split_MB: 272.9 +rank7 [2026-10-28 17:55:19,474] INFO : Step 9600/50000 rank7 | Train Loss: 2.0278 | Train Loss Unweighted: 1.1029 | Train Loss Components: Auxiliary: 0.3926 | Cardinality: 0.2093 | Classification: 0.1886 | Classification_binary: 0.3105 | HitGrouping: 0.2484 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2974 | Regression_eta: 0.0042 | Regression_pt: 0.2980 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.3354 | Regression_cos_phi: 0.1770 | Regression_energy: 0.1420 | Regression_eta: 0.1938 | Regression_pt: 0.1438 | Regression_sin_phi: 0.1513 | LR: 9.21e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.8384s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0193 | max_forward: 1.0416 | max_loss: 0.1771 | max_backward: 1.6304 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6014.9 | uss_MB: 5182.8 | child_rss_MB: 5331.3 | allocated_MB: 345.6 | reserved_MB: 24684.0 | inactive_split_MB: 202.2 +rank7 [2026-10-28 17:59:31,911] INFO : Step 9700/50000 rank7 | Train Loss: 1.9467 | Train Loss Unweighted: 1.0876 | Train Loss Components: Auxiliary: 0.3783 | Cardinality: 0.1959 | Classification: 0.1589 | Classification_binary: 0.3236 | HitGrouping: 0.2489 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2973 | Regression_eta: 0.0037 | Regression_pt: 0.3003 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0968 | Classification_binary: 0.3495 | Regression_cos_phi: 0.1613 | Regression_energy: 0.1666 | Regression_eta: 0.1170 | Regression_pt: 0.1762 | Regression_sin_phi: 0.1680 | LR: 9.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9951s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7431 | max_loss: 0.4417 | max_backward: 1.7429 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6016.7 | uss_MB: 5187.0 | child_rss_MB: 5331.9 | allocated_MB: 346.2 | reserved_MB: 24684.0 | inactive_split_MB: 468.8 +rank7 [2026-10-28 18:03:29,395] INFO : Step 9800/50000 rank7 | Train Loss: 1.6949 | Train Loss Unweighted: 0.9587 | Train Loss Components: Auxiliary: 0.3357 | Cardinality: 0.1712 | Classification: 0.1515 | Classification_binary: 0.2556 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2723 | Regression_eta: 0.0029 | Regression_pt: 0.2733 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2456 | Regression_cos_phi: 0.1290 | Regression_energy: 0.1426 | Regression_eta: 0.0895 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0968 | LR: 9.17e-04 | DataLoad Time: 0.0228s | Model Forward Time: 3.6864s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6504 | max_loss: 0.2354 | max_backward: 1.3239 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.6211 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6013.3 | uss_MB: 5188.1 | child_rss_MB: 5323.9 | allocated_MB: 344.1 | reserved_MB: 24684.0 | inactive_split_MB: 269.9 +rank7 [2026-10-28 18:07:27,800] INFO : Step 9900/50000 rank7 | Train Loss: 1.8753 | Train Loss Unweighted: 1.0347 | Train Loss Components: Auxiliary: 0.3643 | Cardinality: 0.1982 | Classification: 0.1691 | Classification_binary: 0.2834 | HitGrouping: 0.2391 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2852 | Regression_eta: 0.0033 | Regression_pt: 0.2899 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.2670 | Regression_cos_phi: 0.1205 | Regression_energy: 0.1481 | Regression_eta: 0.0859 | Regression_pt: 0.1540 | Regression_sin_phi: 0.1223 | LR: 9.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1757s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0067 | max_forward: 0.8122 | max_loss: 0.1469 | max_backward: 1.7182 | max_optimizer_step: 0.2933 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6005.3 | uss_MB: 5188.8 | child_rss_MB: 5329.3 | allocated_MB: 343.2 | reserved_MB: 25686.0 | inactive_split_MB: 190.8 +rank7 [2026-10-28 18:11:35,686] INFO : Step 10000/50000 rank7 | Train Loss: 2.0467 | Train Loss Unweighted: 1.1161 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.2412 | Classification: 0.1790 | Classification_binary: 0.3408 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2919 | Regression_eta: 0.0036 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1317 | Classification_binary: 0.3272 | Regression_cos_phi: 0.1459 | Regression_energy: 0.1304 | Regression_eta: 0.1273 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1810 | LR: 9.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1269s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7851 | max_loss: 0.1549 | max_backward: 1.7036 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5997.6 | uss_MB: 5192.2 | child_rss_MB: 5322.2 | allocated_MB: 344.7 | reserved_MB: 26692.0 | inactive_split_MB: 294.2 +rank7 [2026-10-28 18:11:36,667] INFO : Running validation on rank7 +rank7 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:11:36,667] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:12:31,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:12:31,988] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 18:12:58,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:12:59,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:13:00,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:13:01,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:13:11,092] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:13:12,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:13:12,690] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 18:13:30,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:13:32,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:13:35,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:13:36,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:13:46,740] INFO : Time taken to make predictions on device 7 is: 0.34 min +rank7 [2026-10-28 18:13:48,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:13:49,360] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 18:14:04,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:14:05,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:14:06,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:14:06,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_10000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:14:16,872] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 18:18:32,574] INFO : Step 10100/50000 rank7 | Train Loss: 1.8401 | Train Loss Unweighted: 0.9837 | Train Loss Components: Auxiliary: 0.3622 | Cardinality: 0.2429 | Classification: 0.1389 | Classification_binary: 0.2788 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2786 | Regression_eta: 0.0031 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2651 | Regression_cos_phi: 0.1056 | Regression_energy: 0.1127 | Regression_eta: 0.1101 | Regression_pt: 0.1156 | Regression_sin_phi: 0.1060 | LR: 9.12e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.2352s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0146 | max_forward: 0.8593 | max_loss: 0.4693 | max_backward: 1.6248 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.4918 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0710 | Memory: rss_MB: 7858.9 | uss_MB: 7014.7 | child_rss_MB: 5313.3 | allocated_MB: 343.3 | reserved_MB: 22350.0 | inactive_split_MB: 245.4 +rank7 [2026-10-28 18:22:34,298] INFO : Step 10200/50000 rank7 | Train Loss: 1.8021 | Train Loss Unweighted: 0.9994 | Train Loss Components: Auxiliary: 0.3488 | Cardinality: 0.1829 | Classification: 0.1663 | Classification_binary: 0.2629 | HitGrouping: 0.2510 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2818 | Regression_eta: 0.0035 | Regression_pt: 0.2815 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.2771 | Regression_cos_phi: 0.1458 | Regression_energy: 0.1309 | Regression_eta: 0.1416 | Regression_pt: 0.1368 | Regression_sin_phi: 0.1489 | LR: 9.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5659s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.1027 | max_forward: 0.9447 | max_loss: 0.1793 | max_backward: 2.0216 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.4806 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7861.4 | uss_MB: 7059.0 | child_rss_MB: 5311.0 | allocated_MB: 343.9 | reserved_MB: 22616.0 | inactive_split_MB: 195.9 +rank7 [2026-10-28 18:26:52,298] INFO : Step 10300/50000 rank7 | Train Loss: 1.9526 | Train Loss Unweighted: 1.0532 | Train Loss Components: Auxiliary: 0.3830 | Cardinality: 0.2452 | Classification: 0.1692 | Classification_binary: 0.3398 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2682 | Regression_eta: 0.0031 | Regression_pt: 0.2694 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.3325 | Regression_cos_phi: 0.1353 | Regression_energy: 0.1183 | Regression_eta: 0.0871 | Regression_pt: 0.1215 | Regression_sin_phi: 0.1098 | LR: 9.09e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.3474s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 1.0714 | max_loss: 0.2272 | max_backward: 2.9985 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6082.9 | uss_MB: 5280.5 | child_rss_MB: 5305.6 | allocated_MB: 347.9 | reserved_MB: 23596.0 | inactive_split_MB: 196.8 +rank7 [2026-10-28 18:31:09,157] INFO : Step 10400/50000 rank7 | Train Loss: 1.7587 | Train Loss Unweighted: 0.9768 | Train Loss Components: Auxiliary: 0.3470 | Cardinality: 0.1794 | Classification: 0.1447 | Classification_binary: 0.2987 | HitGrouping: 0.2445 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2622 | Regression_eta: 0.0030 | Regression_pt: 0.2651 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2822 | Regression_cos_phi: 0.0745 | Regression_energy: 0.1167 | Regression_eta: 0.0915 | Regression_pt: 0.1223 | Regression_sin_phi: 0.1237 | LR: 9.07e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9063s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6848 | max_loss: 0.2495 | max_backward: 1.5818 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0239 | Memory: rss_MB: 6080.7 | uss_MB: 5278.7 | child_rss_MB: 5310.4 | allocated_MB: 341.0 | reserved_MB: 23598.0 | inactive_split_MB: 175.4 +rank7 [2026-10-28 18:35:25,945] INFO : Step 10500/50000 rank7 | Train Loss: 1.6134 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3192 | Cardinality: 0.1788 | Classification: 0.1237 | Classification_binary: 0.2298 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2558 | Regression_eta: 0.0029 | Regression_pt: 0.2580 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.2152 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1207 | Regression_eta: 0.0791 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0602 | LR: 9.06e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.8289s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.6505 | max_loss: 0.2954 | max_backward: 1.4874 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6077.2 | uss_MB: 5279.6 | child_rss_MB: 5305.3 | allocated_MB: 343.2 | reserved_MB: 25580.0 | inactive_split_MB: 206.2 +rank7 [2026-10-28 18:39:26,894] INFO : Step 10600/50000 rank7 | Train Loss: 1.7823 | Train Loss Unweighted: 0.9780 | Train Loss Components: Auxiliary: 0.3581 | Cardinality: 0.1989 | Classification: 0.1580 | Classification_binary: 0.2899 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2604 | Regression_eta: 0.0028 | Regression_pt: 0.2637 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1217 | Regression_eta: 0.0789 | Regression_pt: 0.1262 | Regression_sin_phi: 0.1442 | LR: 9.04e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.2193s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0183 | max_forward: 0.8318 | max_loss: 0.1671 | max_backward: 1.6448 | max_optimizer_step: 0.2870 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6082.0 | uss_MB: 5285.8 | child_rss_MB: 5309.0 | allocated_MB: 347.1 | reserved_MB: 25580.0 | inactive_split_MB: 245.7 +rank7 [2026-10-28 18:43:27,488] INFO : Step 10700/50000 rank7 | Train Loss: 1.8861 | Train Loss Unweighted: 1.0361 | Train Loss Components: Auxiliary: 0.3762 | Cardinality: 0.2303 | Classification: 0.1619 | Classification_binary: 0.3199 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2729 | Regression_eta: 0.0030 | Regression_pt: 0.2751 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2934 | Regression_cos_phi: 0.0960 | Regression_energy: 0.1337 | Regression_eta: 0.0977 | Regression_pt: 0.1395 | Regression_sin_phi: 0.1088 | LR: 9.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2741s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7433 | max_loss: 0.4355 | max_backward: 1.6004 | max_optimizer_step: 0.2779 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6076.2 | uss_MB: 5282.3 | child_rss_MB: 5310.0 | allocated_MB: 346.4 | reserved_MB: 25580.0 | inactive_split_MB: 260.4 +rank7 [2026-10-28 18:47:34,479] INFO : Step 10800/50000 rank7 | Train Loss: 1.5406 | Train Loss Unweighted: 0.8777 | Train Loss Components: Auxiliary: 0.3194 | Cardinality: 0.1557 | Classification: 0.1156 | Classification_binary: 0.2334 | HitGrouping: 0.2383 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0024 | Regression_pt: 0.2623 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2720 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1220 | Regression_eta: 0.0899 | Regression_pt: 0.1251 | Regression_sin_phi: 0.1045 | LR: 9.00e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.3724s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.6507 | max_loss: 0.1475 | max_backward: 1.2226 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6068.2 | uss_MB: 5282.1 | child_rss_MB: 5311.2 | allocated_MB: 341.7 | reserved_MB: 25580.0 | inactive_split_MB: 270.6 +rank7 [2026-10-28 18:51:51,744] INFO : Step 10900/50000 rank7 | Train Loss: 1.7111 | Train Loss Unweighted: 0.9319 | Train Loss Components: Auxiliary: 0.3394 | Cardinality: 0.1954 | Classification: 0.1573 | Classification_binary: 0.2676 | HitGrouping: 0.2332 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2492 | Regression_eta: 0.0028 | Regression_pt: 0.2519 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1049 | Classification_binary: 0.2749 | Regression_cos_phi: 0.1041 | Regression_energy: 0.1460 | Regression_eta: 0.0877 | Regression_pt: 0.1543 | Regression_sin_phi: 0.1278 | LR: 8.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6018s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6010 | max_loss: 0.2461 | max_backward: 1.2359 | max_optimizer_step: 0.2559 | max_tensorboard_logging: 0.6137 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6066.2 | uss_MB: 5286.0 | child_rss_MB: 5316.0 | allocated_MB: 345.2 | reserved_MB: 25580.0 | inactive_split_MB: 196.1 +rank7 [2026-10-28 18:55:59,249] INFO : Step 11000/50000 rank7 | Train Loss: 1.9047 | Train Loss Unweighted: 1.0232 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.2424 | Classification: 0.1647 | Classification_binary: 0.3287 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2614 | Regression_eta: 0.0030 | Regression_pt: 0.2620 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1172 | Classification_binary: 0.3483 | Regression_cos_phi: 0.1384 | Regression_energy: 0.1213 | Regression_eta: 0.1553 | Regression_pt: 0.1259 | Regression_sin_phi: 0.1694 | LR: 8.97e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.8646s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0062 | max_forward: 0.7196 | max_loss: 0.2485 | max_backward: 1.5798 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4775 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6060.2 | uss_MB: 5288.0 | child_rss_MB: 5312.0 | allocated_MB: 345.2 | reserved_MB: 26580.0 | inactive_split_MB: 271.4 +rank7 [2026-10-28 18:56:00,217] INFO : Running validation on rank7 +rank7 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:56:00,217] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:56:54,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:56:55,376] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 18:57:21,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:57:22,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:57:23,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:57:24,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:57:34,794] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:57:35,272] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:57:35,836] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 18:57:52,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:57:53,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:57:55,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:57:56,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:58:06,262] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 18:58:06,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:58:06,770] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 18:58:21,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:58:22,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:58:23,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:58:23,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_11000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:58:33,736] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 19:03:04,099] INFO : Step 11100/50000 rank7 | Train Loss: 1.6416 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3332 | Cardinality: 0.2007 | Classification: 0.1358 | Classification_binary: 0.2689 | HitGrouping: 0.2355 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2408 | Regression_eta: 0.0023 | Regression_pt: 0.2424 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2977 | Regression_cos_phi: 0.1182 | Regression_energy: 0.1183 | Regression_eta: 0.0912 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1240 | LR: 8.95e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.4746s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8357 | max_loss: 0.4229 | max_backward: 1.3411 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 1.1199 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 7808.5 | uss_MB: 7037.1 | child_rss_MB: 5345.3 | allocated_MB: 341.2 | reserved_MB: 23114.0 | inactive_split_MB: 247.8 +rank7 [2026-10-28 19:07:10,329] INFO : Step 11200/50000 rank7 | Train Loss: 1.8658 | Train Loss Unweighted: 0.9376 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.3117 | Classification: 0.1324 | Classification_binary: 0.2309 | HitGrouping: 0.2441 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2804 | Regression_eta: 0.0036 | Regression_pt: 0.2873 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2443 | Regression_cos_phi: 0.1381 | Regression_energy: 0.1495 | Regression_eta: 0.1206 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0989 | LR: 8.93e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8081s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0235 | max_forward: 0.6441 | max_loss: 0.1420 | max_backward: 1.4542 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5471 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 7795.2 | uss_MB: 7039.6 | child_rss_MB: 5321.4 | allocated_MB: 343.6 | reserved_MB: 23114.0 | inactive_split_MB: 207.3 +rank7 [2026-10-28 19:11:04,859] INFO : Step 11300/50000 rank7 | Train Loss: 1.9126 | Train Loss Unweighted: 0.9698 | Train Loss Components: Auxiliary: 0.3808 | Cardinality: 0.3154 | Classification: 0.1453 | Classification_binary: 0.3032 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2568 | Regression_eta: 0.0028 | Regression_pt: 0.2584 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.3138 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1072 | Regression_eta: 0.0876 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1120 | LR: 8.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2512s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0070 | max_forward: 0.7813 | max_loss: 0.2533 | max_backward: 1.8436 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6121.4 | uss_MB: 5367.0 | child_rss_MB: 5333.6 | allocated_MB: 345.2 | reserved_MB: 23704.0 | inactive_split_MB: 247.4 +rank7 [2026-10-28 19:14:57,052] INFO : Step 11400/50000 rank7 | Train Loss: 1.7616 | Train Loss Unweighted: 0.9603 | Train Loss Components: Auxiliary: 0.3528 | Cardinality: 0.1990 | Classification: 0.1266 | Classification_binary: 0.3042 | HitGrouping: 0.2390 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2600 | Regression_eta: 0.0032 | Regression_pt: 0.2634 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2978 | Regression_cos_phi: 0.1016 | Regression_energy: 0.1218 | Regression_eta: 0.0871 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0991 | LR: 8.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0020s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7284 | max_loss: 0.1519 | max_backward: 1.5400 | max_optimizer_step: 0.1864 | max_tensorboard_logging: 0.6162 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6119.4 | uss_MB: 5366.6 | child_rss_MB: 5341.0 | allocated_MB: 343.9 | reserved_MB: 25002.0 | inactive_split_MB: 259.5 +rank7 [2026-10-28 19:18:58,942] INFO : Step 11500/50000 rank7 | Train Loss: 1.5617 | Train Loss Unweighted: 0.8672 | Train Loss Components: Auxiliary: 0.3107 | Cardinality: 0.1714 | Classification: 0.1394 | Classification_binary: 0.2303 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2456 | Regression_eta: 0.0025 | Regression_pt: 0.2467 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1018 | Classification_binary: 0.2375 | Regression_cos_phi: 0.1110 | Regression_energy: 0.1321 | Regression_eta: 0.1100 | Regression_pt: 0.1384 | Regression_sin_phi: 0.1299 | LR: 8.88e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.4525s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.5683 | max_loss: 0.1393 | max_backward: 1.1727 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6122.1 | uss_MB: 5369.8 | child_rss_MB: 5318.4 | allocated_MB: 343.6 | reserved_MB: 26000.0 | inactive_split_MB: 166.8 +rank7 [2026-10-28 19:22:59,322] INFO : Step 11600/50000 rank7 | Train Loss: 1.7165 | Train Loss Unweighted: 0.9272 | Train Loss Components: Auxiliary: 0.3452 | Cardinality: 0.2011 | Classification: 0.1544 | Classification_binary: 0.2772 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2444 | Regression_eta: 0.0030 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1283 | Regression_energy: 0.1040 | Regression_eta: 0.1160 | Regression_pt: 0.1089 | Regression_sin_phi: 0.1185 | LR: 8.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8313s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.6560 | max_loss: 0.2455 | max_backward: 1.3498 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.6667 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6096.5 | uss_MB: 5368.8 | child_rss_MB: 5323.9 | allocated_MB: 346.4 | reserved_MB: 26000.0 | inactive_split_MB: 223.1 +rank7 [2026-10-28 19:26:57,456] INFO : Step 11700/50000 rank7 | Train Loss: 1.7899 | Train Loss Unweighted: 0.9443 | Train Loss Components: Auxiliary: 0.3645 | Cardinality: 0.2468 | Classification: 0.1474 | Classification_binary: 0.3085 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0027 | Regression_pt: 0.2415 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.3159 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1259 | Regression_eta: 0.1016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.1103 | LR: 8.84e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1774s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7754 | max_loss: 0.1801 | max_backward: 1.7182 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.6140 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6099.8 | uss_MB: 5368.9 | child_rss_MB: 5328.0 | allocated_MB: 347.7 | reserved_MB: 26000.0 | inactive_split_MB: 422.1 +rank7 [2026-10-28 19:30:54,021] INFO : Step 11800/50000 rank7 | Train Loss: 1.6037 | Train Loss Unweighted: 0.8588 | Train Loss Components: Auxiliary: 0.3214 | Cardinality: 0.1816 | Classification: 0.1403 | Classification_binary: 0.2419 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2345 | Regression_eta: 0.0027 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2666 | Regression_cos_phi: 0.1506 | Regression_energy: 0.1202 | Regression_eta: 0.1071 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0763 | LR: 8.82e-04 | DataLoad Time: 0.0242s | Model Forward Time: 6.0636s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0124 | max_forward: 1.0726 | max_loss: 0.3247 | max_backward: 2.6119 | max_optimizer_step: 0.1988 | max_tensorboard_logging: 0.9282 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6101.3 | uss_MB: 4317.4 | child_rss_MB: 5352.5 | allocated_MB: 342.4 | reserved_MB: 26000.0 | inactive_split_MB: 255.5 +rank7 [2026-10-28 19:34:55,960] INFO : Step 11900/50000 rank7 | Train Loss: 1.6781 | Train Loss Unweighted: 0.8768 | Train Loss Components: Auxiliary: 0.3319 | Cardinality: 0.2120 | Classification: 0.1495 | Classification_binary: 0.2522 | HitGrouping: 0.2320 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2332 | Regression_eta: 0.0028 | Regression_pt: 0.2360 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0807 | Regression_energy: 0.1066 | Regression_eta: 0.0779 | Regression_pt: 0.1112 | Regression_sin_phi: 0.1004 | LR: 8.80e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9519s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8855 | max_loss: 0.2998 | max_backward: 1.9950 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 1.0725 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 6093.1 | uss_MB: 3984.3 | child_rss_MB: 5329.1 | allocated_MB: 344.3 | reserved_MB: 26144.0 | inactive_split_MB: 165.6 +rank7 [2026-10-28 19:39:04,011] INFO : Step 12000/50000 rank7 | Train Loss: 1.8417 | Train Loss Unweighted: 0.9564 | Train Loss Components: Auxiliary: 0.3760 | Cardinality: 0.2648 | Classification: 0.1487 | Classification_binary: 0.3148 | HitGrouping: 0.2321 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2433 | Regression_eta: 0.0028 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1200 | Classification_binary: 0.3440 | Regression_cos_phi: 0.1479 | Regression_energy: 0.1216 | Regression_eta: 0.1393 | Regression_pt: 0.1264 | Regression_sin_phi: 0.1334 | LR: 8.78e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3551s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.7751 | max_loss: 0.1567 | max_backward: 1.6164 | max_optimizer_step: 0.2862 | max_tensorboard_logging: 0.7907 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6086.3 | uss_MB: 5375.5 | child_rss_MB: 5354.6 | allocated_MB: 349.3 | reserved_MB: 27032.0 | inactive_split_MB: 281.2 +rank7 [2026-10-28 19:39:04,976] INFO : Running validation on rank7 +rank7 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 19:39:04,976] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 19:39:55,051] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:39:55,582] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 19:40:21,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 19:40:22,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 19:40:23,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 19:40:24,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 19:40:34,277] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:40:34,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:40:35,410] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 19:40:51,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 19:40:52,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 19:40:53,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 19:40:54,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 19:41:04,885] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:41:05,105] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:41:05,609] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 19:41:20,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 19:41:21,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 19:41:21,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 19:41:22,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_12000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 19:41:32,588] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 19:45:57,644] INFO : Step 12100/50000 rank7 | Train Loss: 1.5643 | Train Loss Unweighted: 0.8417 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1812 | Classification: 0.1222 | Classification_binary: 0.2620 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2251 | Regression_eta: 0.0025 | Regression_pt: 0.2272 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2393 | Regression_cos_phi: 0.0808 | Regression_energy: 0.1063 | Regression_eta: 0.0576 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0733 | LR: 8.76e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8588s | Phase Times: max_pre_step_barrier: 0.1095 | max_batch_to_device: 0.0064 | max_forward: 0.8614 | max_loss: 0.4724 | max_backward: 1.9276 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.9007 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7779.0 | uss_MB: 7074.2 | child_rss_MB: 5324.2 | allocated_MB: 347.3 | reserved_MB: 21864.0 | inactive_split_MB: 201.5 +rank7 [2026-10-28 19:50:02,389] INFO : Step 12200/50000 rank7 | Train Loss: 1.5774 | Train Loss Unweighted: 0.8379 | Train Loss Components: Auxiliary: 0.3232 | Cardinality: 0.1972 | Classification: 0.1378 | Classification_binary: 0.2311 | HitGrouping: 0.2420 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2322 | Regression_eta: 0.0024 | Regression_pt: 0.2319 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2545 | Regression_cos_phi: 0.1236 | Regression_energy: 0.1069 | Regression_eta: 0.1158 | Regression_pt: 0.1122 | Regression_sin_phi: 0.1206 | LR: 8.74e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.9624s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0071 | max_forward: 0.7162 | max_loss: 0.4335 | max_backward: 1.2790 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 0.8015 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1044 | Memory: rss_MB: 7780.2 | uss_MB: 7078.9 | child_rss_MB: 5305.9 | allocated_MB: 344.4 | reserved_MB: 22858.0 | inactive_split_MB: 240.3 +rank7 [2026-10-28 19:54:00,334] INFO : Step 12300/50000 rank7 | Train Loss: 1.8368 | Train Loss Unweighted: 0.9284 | Train Loss Components: Auxiliary: 0.3676 | Cardinality: 0.2914 | Classification: 0.1605 | Classification_binary: 0.2913 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2342 | Regression_eta: 0.0029 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2995 | Regression_cos_phi: 0.1239 | Regression_energy: 0.1262 | Regression_eta: 0.1032 | Regression_pt: 0.1331 | Regression_sin_phi: 0.1140 | LR: 8.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4610s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7472 | max_loss: 0.4145 | max_backward: 1.8009 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.9019 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6188.3 | uss_MB: 5484.2 | child_rss_MB: 5326.8 | allocated_MB: 344.4 | reserved_MB: 23876.0 | inactive_split_MB: 200.3 +rank7 [2026-10-28 19:58:08,174] INFO : Step 12400/50000 rank7 | Train Loss: 1.6525 | Train Loss Unweighted: 0.8928 | Train Loss Components: Auxiliary: 0.3298 | Cardinality: 0.1844 | Classification: 0.1285 | Classification_binary: 0.2994 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2296 | Regression_eta: 0.0026 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2762 | Regression_cos_phi: 0.1294 | Regression_energy: 0.1217 | Regression_eta: 0.0805 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1367 | LR: 8.70e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.9472s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0145 | max_forward: 1.3854 | max_loss: 0.1608 | max_backward: 2.1101 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.8992 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6183.2 | uss_MB: 5487.0 | child_rss_MB: 5334.4 | allocated_MB: 343.7 | reserved_MB: 24870.0 | inactive_split_MB: 260.3 +rank7 [2026-10-28 20:02:34,177] INFO : Step 12500/50000 rank7 | Train Loss: 1.4220 | Train Loss Unweighted: 0.7647 | Train Loss Components: Auxiliary: 0.2837 | Cardinality: 0.1505 | Classification: 0.1052 | Classification_binary: 0.2180 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2185 | Regression_eta: 0.0022 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.2389 | Regression_cos_phi: 0.1048 | Regression_energy: 0.1034 | Regression_eta: 0.0765 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0865 | LR: 8.68e-04 | DataLoad Time: 0.0239s | Model Forward Time: 4.6929s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.6169 | max_loss: 0.3346 | max_backward: 1.9995 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.7819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6175.9 | uss_MB: 5484.1 | child_rss_MB: 5301.2 | allocated_MB: 342.4 | reserved_MB: 24870.0 | inactive_split_MB: 242.8 +rank7 [2026-10-28 20:06:38,789] INFO : Step 12600/50000 rank7 | Train Loss: 1.5956 | Train Loss Unweighted: 0.8460 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1795 | Classification: 0.1317 | Classification_binary: 0.2538 | HitGrouping: 0.2290 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2270 | Regression_eta: 0.0026 | Regression_pt: 0.2280 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1980 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0855 | Regression_eta: 0.0714 | Regression_pt: 0.0870 | Regression_sin_phi: 0.1008 | LR: 8.66e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.9523s | Phase Times: max_pre_step_barrier: 0.0124 | max_batch_to_device: 0.0747 | max_forward: 1.7920 | max_loss: 0.2343 | max_backward: 2.2383 | max_optimizer_step: 0.2858 | max_tensorboard_logging: 0.7889 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6181.8 | uss_MB: 5491.9 | child_rss_MB: 5334.1 | allocated_MB: 347.9 | reserved_MB: 24940.0 | inactive_split_MB: 206.3 +rank7 [2026-10-28 20:10:45,235] INFO : Step 12700/50000 rank7 | Train Loss: 1.6797 | Train Loss Unweighted: 0.8725 | Train Loss Components: Auxiliary: 0.3358 | Cardinality: 0.2224 | Classification: 0.1332 | Classification_binary: 0.2918 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2197 | Regression_eta: 0.0026 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2942 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1069 | Regression_eta: 0.1051 | Regression_pt: 0.1117 | Regression_sin_phi: 0.1273 | LR: 8.64e-04 | DataLoad Time: 0.0116s | Model Forward Time: 5.2285s | Phase Times: max_pre_step_barrier: 0.0413 | max_batch_to_device: 0.0180 | max_forward: 2.0308 | max_loss: 0.5205 | max_backward: 2.0188 | max_optimizer_step: 0.1896 | max_tensorboard_logging: 0.5011 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6168.1 | uss_MB: 5488.1 | child_rss_MB: 5334.5 | allocated_MB: 343.0 | reserved_MB: 24940.0 | inactive_split_MB: 275.1 +rank7 [2026-10-28 20:14:52,958] INFO : Step 12800/50000 rank7 | Train Loss: 1.5560 | Train Loss Unweighted: 0.8349 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1804 | Classification: 0.1200 | Classification_binary: 0.2546 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2253 | Regression_eta: 0.0024 | Regression_pt: 0.2300 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2607 | Regression_cos_phi: 0.0925 | Regression_energy: 0.1115 | Regression_eta: 0.0609 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0738 | LR: 8.62e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.8535s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6930 | max_loss: 0.2517 | max_backward: 1.4462 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.6155 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6161.2 | uss_MB: 5493.0 | child_rss_MB: 5311.0 | allocated_MB: 342.9 | reserved_MB: 24942.0 | inactive_split_MB: 205.5 +rank7 [2026-10-28 20:19:01,948] INFO : Step 12900/50000 rank7 | Train Loss: 1.5239 | Train Loss Unweighted: 0.8160 | Train Loss Components: Auxiliary: 0.3087 | Cardinality: 0.1708 | Classification: 0.1377 | Classification_binary: 0.2382 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2184 | Regression_eta: 0.0023 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2246 | Regression_cos_phi: 0.0952 | Regression_energy: 0.1011 | Regression_eta: 0.0721 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1210 | LR: 8.60e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.5187s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.6007 | max_loss: 0.2098 | max_backward: 1.0668 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6155.7 | uss_MB: 5489.6 | child_rss_MB: 5333.1 | allocated_MB: 344.7 | reserved_MB: 24942.0 | inactive_split_MB: 216.5 +rank7 [2026-10-28 20:23:25,504] INFO : Step 13000/50000 rank7 | Train Loss: 1.7741 | Train Loss Unweighted: 0.8792 | Train Loss Components: Auxiliary: 0.3547 | Cardinality: 0.2714 | Classification: 0.1382 | Classification_binary: 0.2990 | HitGrouping: 0.2288 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2162 | Regression_eta: 0.0028 | Regression_pt: 0.2199 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2898 | Regression_cos_phi: 0.0922 | Regression_energy: 0.1006 | Regression_eta: 0.0955 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0805 | LR: 8.58e-04 | DataLoad Time: 0.0152s | Model Forward Time: 10.3381s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.2218 | max_forward: 0.9481 | max_loss: 0.4786 | max_backward: 1.8881 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 5.9885 | max_tensorboard_flush: 0.0067 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6158.7 | uss_MB: 4830.2 | child_rss_MB: 5330.8 | allocated_MB: 347.1 | reserved_MB: 25962.0 | inactive_split_MB: 252.9 +rank7 [2026-10-28 20:23:26,442] INFO : Running validation on rank7 +rank7 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 20:23:26,443] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 20:24:18,600] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:24:19,151] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 20:24:45,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 20:24:46,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 20:24:47,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 20:24:48,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 20:24:58,378] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 20:25:04,185] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:25:04,756] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 20:25:20,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 20:25:21,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 20:25:22,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 20:25:23,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 20:25:33,310] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 20:25:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:25:34,723] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 20:25:50,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 20:25:51,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 20:25:52,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 20:25:52,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_13000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 20:26:02,710] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 20:30:28,056] INFO : Step 13100/50000 rank7 | Train Loss: 1.5665 | Train Loss Unweighted: 0.8057 | Train Loss Components: Auxiliary: 0.3203 | Cardinality: 0.2199 | Classification: 0.1129 | Classification_binary: 0.2553 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2148 | Regression_eta: 0.0023 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2738 | Regression_cos_phi: 0.0950 | Regression_energy: 0.1091 | Regression_eta: 0.0834 | Regression_pt: 0.1122 | Regression_sin_phi: 0.1106 | LR: 8.56e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0205s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0116 | max_forward: 0.8108 | max_loss: 0.4471 | max_backward: 1.7956 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.8962 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7814.2 | uss_MB: 5678.6 | child_rss_MB: 5332.5 | allocated_MB: 346.0 | reserved_MB: 25506.0 | inactive_split_MB: 197.3 +rank7 [2026-10-28 20:34:38,789] INFO : Step 13200/50000 rank7 | Train Loss: 1.4224 | Train Loss Unweighted: 0.7569 | Train Loss Components: Auxiliary: 0.2897 | Cardinality: 0.1600 | Classification: 0.1187 | Classification_binary: 0.2156 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2084 | Regression_eta: 0.0024 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0556 | Regression_energy: 0.1107 | Regression_eta: 0.0593 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0636 | LR: 8.54e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.0414s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7439 | max_loss: 0.1414 | max_backward: 1.3946 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.7653 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7815.2 | uss_MB: 6435.0 | child_rss_MB: 5304.2 | allocated_MB: 341.3 | reserved_MB: 26548.0 | inactive_split_MB: 252.6 +rank7 [2026-10-28 20:38:50,943] INFO : Step 13300/50000 rank7 | Train Loss: 1.5784 | Train Loss Unweighted: 0.8348 | Train Loss Components: Auxiliary: 0.3168 | Cardinality: 0.1767 | Classification: 0.1324 | Classification_binary: 0.2645 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2163 | Regression_eta: 0.0028 | Regression_pt: 0.2162 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2614 | Regression_cos_phi: 0.0833 | Regression_energy: 0.1122 | Regression_eta: 0.0623 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0593 | LR: 8.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.4799s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.1648 | max_loss: 0.2215 | max_backward: 2.8107 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.5451 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0722 | Memory: rss_MB: 6241.1 | uss_MB: 4586.2 | child_rss_MB: 5309.3 | allocated_MB: 346.3 | reserved_MB: 26548.0 | inactive_split_MB: 201.7 +rank7 [2026-10-28 20:43:04,348] INFO : Step 13400/50000 rank7 | Train Loss: 1.6355 | Train Loss Unweighted: 0.8286 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2223 | Classification: 0.1286 | Classification_binary: 0.2899 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2015 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2911 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0922 | Regression_eta: 0.0825 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0947 | LR: 8.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2851s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0202 | max_forward: 0.7908 | max_loss: 0.1537 | max_backward: 1.7907 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5416 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6235.1 | uss_MB: 5592.6 | child_rss_MB: 5332.9 | allocated_MB: 347.5 | reserved_MB: 26548.0 | inactive_split_MB: 266.6 +rank7 [2026-10-28 20:47:13,648] INFO : Step 13500/50000 rank7 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7332 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1734 | Classification: 0.1091 | Classification_binary: 0.2184 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2006 | Regression_eta: 0.0022 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2080 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0920 | Regression_eta: 0.0626 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0892 | LR: 8.48e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.1207s | Phase Times: max_pre_step_barrier: 0.0076 | max_batch_to_device: 0.0474 | max_forward: 0.7865 | max_loss: 0.1618 | max_backward: 1.5873 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.6181 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6231.2 | uss_MB: 5580.0 | child_rss_MB: 5341.8 | allocated_MB: 343.7 | reserved_MB: 26548.0 | inactive_split_MB: 183.2 +rank7 [2026-10-28 20:51:26,251] INFO : Step 13600/50000 rank7 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7720 | Train Loss Components: Auxiliary: 0.3018 | Cardinality: 0.1797 | Classification: 0.1189 | Classification_binary: 0.2356 | HitGrouping: 0.2232 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0024 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2445 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1444 | Regression_eta: 0.0726 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0946 | LR: 8.45e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.8039s | Phase Times: max_pre_step_barrier: 0.0146 | max_batch_to_device: 0.0067 | max_forward: 0.6622 | max_loss: 0.2531 | max_backward: 1.4302 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.5980 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6230.5 | uss_MB: 5596.5 | child_rss_MB: 5295.7 | allocated_MB: 346.4 | reserved_MB: 26548.0 | inactive_split_MB: 165.6 +rank7 [2026-10-28 20:55:41,374] INFO : Step 13700/50000 rank7 | Train Loss: 1.5585 | Train Loss Unweighted: 0.8168 | Train Loss Components: Auxiliary: 0.3176 | Cardinality: 0.1904 | Classification: 0.1227 | Classification_binary: 0.2969 | HitGrouping: 0.2198 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1950 | Regression_eta: 0.0022 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2561 | Regression_cos_phi: 0.0859 | Regression_energy: 0.1031 | Regression_eta: 0.0686 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0698 | LR: 8.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8071s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8407 | max_loss: 0.1552 | max_backward: 1.4690 | max_optimizer_step: 0.2522 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 6227.1 | uss_MB: 4467.2 | child_rss_MB: 5339.6 | allocated_MB: 348.2 | reserved_MB: 26548.0 | inactive_split_MB: 277.8 +rank7 [2026-10-28 20:59:54,240] INFO : Step 13800/50000 rank7 | Train Loss: 1.4089 | Train Loss Unweighted: 0.7593 | Train Loss Components: Auxiliary: 0.2834 | Cardinality: 0.1353 | Classification: 0.1053 | Classification_binary: 0.2383 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2050 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2274 | Regression_cos_phi: 0.0791 | Regression_energy: 0.1043 | Regression_eta: 0.0679 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0871 | LR: 8.41e-04 | DataLoad Time: 0.0226s | Model Forward Time: 6.1135s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0131 | max_forward: 0.7986 | max_loss: 0.1805 | max_backward: 3.3956 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.8967 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6229.1 | uss_MB: 5595.4 | child_rss_MB: 5342.0 | allocated_MB: 343.9 | reserved_MB: 26672.0 | inactive_split_MB: 264.6 +rank7 [2026-10-28 21:04:03,369] INFO : Step 13900/50000 rank7 | Train Loss: 1.3403 | Train Loss Unweighted: 0.7324 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1350 | Classification: 0.1240 | Classification_binary: 0.2048 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1993 | Regression_eta: 0.0020 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0941 | Classification_binary: 0.2554 | Regression_cos_phi: 0.1239 | Regression_energy: 0.1128 | Regression_eta: 0.1161 | Regression_pt: 0.1196 | Regression_sin_phi: 0.1250 | LR: 8.39e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.1586s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6925 | max_loss: 0.4310 | max_backward: 1.5374 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.2229 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1045 | Memory: rss_MB: 6234.1 | uss_MB: 5390.0 | child_rss_MB: 5334.5 | allocated_MB: 344.8 | reserved_MB: 26672.0 | inactive_split_MB: 211.9 +rank7 [2026-10-28 21:08:24,062] INFO : Step 14000/50000 rank7 | Train Loss: 1.6232 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1757 | Classification: 0.1510 | Classification_binary: 0.2791 | HitGrouping: 0.2258 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2083 | Regression_eta: 0.0027 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2911 | Regression_cos_phi: 0.0934 | Regression_energy: 0.1017 | Regression_eta: 0.0837 | Regression_pt: 0.1086 | Regression_sin_phi: 0.1107 | LR: 8.37e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.9206s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0313 | max_forward: 0.8109 | max_loss: 0.1567 | max_backward: 2.5369 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6233.4 | uss_MB: 5470.4 | child_rss_MB: 5334.3 | allocated_MB: 345.3 | reserved_MB: 26672.0 | inactive_split_MB: 262.2 +rank7 [2026-10-28 21:08:24,998] INFO : Running validation on rank7 +rank7 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:08:24,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:09:16,436] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:09:16,960] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 21:09:42,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:09:43,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:09:44,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:09:45,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:09:55,833] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:09:56,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:09:57,000] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 21:10:13,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:10:14,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:10:15,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:10:16,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:10:26,217] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 21:10:26,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:10:26,832] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 21:10:42,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:10:43,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:10:44,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:10:46,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_14000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:10:56,329] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:15:28,843] INFO : Step 14100/50000 rank7 | Train Loss: 1.4834 | Train Loss Unweighted: 0.7641 | Train Loss Components: Auxiliary: 0.2993 | Cardinality: 0.1689 | Classification: 0.1196 | Classification_binary: 0.2641 | HitGrouping: 0.2153 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1870 | Regression_eta: 0.0024 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2638 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0856 | Regression_eta: 0.1029 | Regression_pt: 0.0887 | Regression_sin_phi: 0.1285 | LR: 8.35e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5866s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0949 | max_forward: 0.8812 | max_loss: 0.4365 | max_backward: 1.7061 | max_optimizer_step: 0.2224 | max_tensorboard_logging: 0.7046 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7903.8 | uss_MB: 7266.8 | child_rss_MB: 5327.0 | allocated_MB: 345.6 | reserved_MB: 26508.0 | inactive_split_MB: 404.1 +rank7 [2026-10-28 21:19:48,614] INFO : Step 14200/50000 rank7 | Train Loss: 1.2845 | Train Loss Unweighted: 0.6727 | Train Loss Components: Auxiliary: 0.2610 | Cardinality: 0.1427 | Classification: 0.0968 | Classification_binary: 0.1979 | HitGrouping: 0.2157 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1868 | Regression_eta: 0.0021 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.2192 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0919 | Regression_eta: 0.1041 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0660 | LR: 8.32e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4850s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0154 | max_forward: 0.9264 | max_loss: 0.3967 | max_backward: 1.7025 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7240 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 7903.8 | uss_MB: 7270.8 | child_rss_MB: 5324.3 | allocated_MB: 340.1 | reserved_MB: 27568.0 | inactive_split_MB: 219.0 +rank7 [2026-10-28 21:24:03,127] INFO : Step 14300/50000 rank7 | Train Loss: 1.3940 | Train Loss Unweighted: 0.7288 | Train Loss Components: Auxiliary: 0.2807 | Cardinality: 0.1568 | Classification: 0.1170 | Classification_binary: 0.2325 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1864 | Regression_eta: 0.0021 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2179 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0910 | Regression_eta: 0.0563 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0608 | LR: 8.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0227s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6454 | max_loss: 0.2489 | max_backward: 1.3745 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8520 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0380 | Memory: rss_MB: 6341.2 | uss_MB: 5709.7 | child_rss_MB: 5298.6 | allocated_MB: 344.8 | reserved_MB: 27568.0 | inactive_split_MB: 228.4 +rank7 [2026-10-28 21:28:09,727] INFO : Step 14400/50000 rank7 | Train Loss: 1.5568 | Train Loss Unweighted: 0.7979 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.1891 | Classification: 0.1141 | Classification_binary: 0.2895 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1937 | Regression_eta: 0.0024 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2777 | Regression_cos_phi: 0.1006 | Regression_energy: 0.0962 | Regression_eta: 0.0952 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0804 | LR: 8.28e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0262s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7639 | max_loss: 0.1531 | max_backward: 1.5148 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6343.2 | uss_MB: 5711.5 | child_rss_MB: 5323.7 | allocated_MB: 346.3 | reserved_MB: 27568.0 | inactive_split_MB: 274.5 +rank7 [2026-10-28 21:32:11,132] INFO : Step 14500/50000 rank7 | Train Loss: 1.3080 | Train Loss Unweighted: 0.6786 | Train Loss Components: Auxiliary: 0.2668 | Cardinality: 0.1483 | Classification: 0.0944 | Classification_binary: 0.2142 | HitGrouping: 0.2123 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1837 | Regression_eta: 0.0020 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2233 | Regression_cos_phi: 0.1136 | Regression_energy: 0.0958 | Regression_eta: 0.0778 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0900 | LR: 8.26e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.3471s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0060 | max_forward: 0.6421 | max_loss: 0.3217 | max_backward: 2.8122 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.8164 | max_tensorboard_flush: 0.0034 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6339.3 | uss_MB: 4858.2 | child_rss_MB: 5325.7 | allocated_MB: 340.6 | reserved_MB: 27568.0 | inactive_split_MB: 220.9 +rank7 [2026-10-28 21:36:11,051] INFO : Step 14600/50000 rank7 | Train Loss: 1.4140 | Train Loss Unweighted: 0.7318 | Train Loss Components: Auxiliary: 0.2878 | Cardinality: 0.1677 | Classification: 0.1155 | Classification_binary: 0.2228 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0886 | Regression_eta: 0.0610 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0799 | LR: 8.24e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.9766s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0212 | max_forward: 0.7198 | max_loss: 0.1489 | max_backward: 1.2802 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.9068 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6342.3 | uss_MB: 5713.4 | child_rss_MB: 5312.8 | allocated_MB: 341.2 | reserved_MB: 27568.0 | inactive_split_MB: 236.9 +rank7 [2026-10-28 21:40:16,607] INFO : Step 14700/50000 rank7 | Train Loss: 1.6078 | Train Loss Unweighted: 0.8225 | Train Loss Components: Auxiliary: 0.3286 | Cardinality: 0.2194 | Classification: 0.1320 | Classification_binary: 0.2949 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0024 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2827 | Regression_cos_phi: 0.1249 | Regression_energy: 0.0905 | Regression_eta: 0.0757 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0901 | LR: 8.21e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.1352s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0075 | max_forward: 1.0875 | max_loss: 0.2551 | max_backward: 2.0668 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.7940 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.1225 | Memory: rss_MB: 6339.3 | uss_MB: 5714.0 | child_rss_MB: 5304.9 | allocated_MB: 346.0 | reserved_MB: 27568.0 | inactive_split_MB: 416.0 +rank7 [2026-10-28 21:44:33,910] INFO : Step 14800/50000 rank7 | Train Loss: 1.4030 | Train Loss Unweighted: 0.7487 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1472 | Classification: 0.1082 | Classification_binary: 0.2219 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2055 | Regression_eta: 0.0021 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2564 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0912 | Regression_eta: 0.0730 | Regression_pt: 0.0949 | Regression_sin_phi: 0.1022 | LR: 8.19e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3506s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0144 | max_forward: 1.0042 | max_loss: 0.2979 | max_backward: 1.7642 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5284 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 6340.4 | uss_MB: 5714.7 | child_rss_MB: 5318.2 | allocated_MB: 344.6 | reserved_MB: 27568.0 | inactive_split_MB: 292.7 +rank7 [2026-10-28 21:48:49,149] INFO : Step 14900/50000 rank7 | Train Loss: 1.2126 | Train Loss Unweighted: 0.6517 | Train Loss Components: Auxiliary: 0.2536 | Cardinality: 0.1184 | Classification: 0.1002 | Classification_binary: 0.1874 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0906 | Regression_energy: 0.0817 | Regression_eta: 0.0584 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0749 | LR: 8.17e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.3236s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.5839 | max_loss: 0.1425 | max_backward: 1.0892 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4928 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6334.4 | uss_MB: 5713.4 | child_rss_MB: 5297.4 | allocated_MB: 339.9 | reserved_MB: 27568.0 | inactive_split_MB: 141.3 +rank7 [2026-10-28 21:52:53,293] INFO : Step 15000/50000 rank7 | Train Loss: 1.5014 | Train Loss Unweighted: 0.7740 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1952 | Classification: 0.1181 | Classification_binary: 0.2399 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2047 | Regression_eta: 0.0024 | Regression_pt: 0.2065 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2156 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0880 | Regression_eta: 0.0686 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0712 | LR: 8.15e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7265s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6874 | max_loss: 0.1459 | max_backward: 1.4383 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.5018 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6334.4 | uss_MB: 5719.2 | child_rss_MB: 5317.9 | allocated_MB: 343.7 | reserved_MB: 27568.0 | inactive_split_MB: 200.9 +rank7 [2026-10-28 21:52:54,210] INFO : Running validation on rank7 +rank7 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:52:54,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:53:44,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:53:44,569] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 21:54:10,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:54:11,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:54:12,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:54:13,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:54:23,794] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:54:24,123] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:54:24,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 21:54:41,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:54:41,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:54:42,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:54:43,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:54:53,903] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:54:54,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:54:55,062] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 21:55:10,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:55:11,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:55:12,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:55:12,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_15000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:55:22,968] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 21:59:40,818] INFO : Step 15100/50000 rank7 | Train Loss: 1.3762 | Train Loss Unweighted: 0.7255 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1317 | Classification: 0.1082 | Classification_binary: 0.2450 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1822 | Regression_eta: 0.0026 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2510 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0883 | Regression_eta: 0.1170 | Regression_pt: 0.0935 | Regression_sin_phi: 0.1052 | LR: 8.12e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.6881s | Phase Times: max_pre_step_barrier: 0.0073 | max_batch_to_device: 0.0204 | max_forward: 1.1099 | max_loss: 0.3310 | max_backward: 2.5681 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.8920 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 7870.2 | uss_MB: 7246.0 | child_rss_MB: 5336.8 | allocated_MB: 345.0 | reserved_MB: 22818.0 | inactive_split_MB: 460.2 +rank7 [2026-10-28 22:03:40,545] INFO : Step 15200/50000 rank7 | Train Loss: 1.3639 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2706 | Cardinality: 0.1591 | Classification: 0.1142 | Classification_binary: 0.1959 | HitGrouping: 0.2190 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1853 | Regression_eta: 0.0024 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2053 | Regression_cos_phi: 0.0947 | Regression_energy: 0.0812 | Regression_eta: 0.0631 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0961 | LR: 8.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9625s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.1006 | max_forward: 1.0452 | max_loss: 0.2115 | max_backward: 1.7550 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.7998 | max_tensorboard_flush: 0.0131 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7869.7 | uss_MB: 6097.2 | child_rss_MB: 5331.0 | allocated_MB: 342.2 | reserved_MB: 23588.0 | inactive_split_MB: 198.6 +rank7 [2026-10-28 22:07:35,577] INFO : Step 15300/50000 rank7 | Train Loss: 1.3875 | Train Loss Unweighted: 0.6957 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1701 | Classification: 0.1168 | Classification_binary: 0.2259 | HitGrouping: 0.2158 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1744 | Regression_eta: 0.0021 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2238 | Regression_cos_phi: 0.1539 | Regression_energy: 0.0891 | Regression_eta: 0.0955 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1106 | LR: 8.08e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8193s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0734 | max_forward: 0.8308 | max_loss: 0.1512 | max_backward: 1.4865 | max_optimizer_step: 0.2977 | max_tensorboard_logging: 1.0032 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6365.8 | uss_MB: 5643.8 | child_rss_MB: 5332.2 | allocated_MB: 346.0 | reserved_MB: 23588.0 | inactive_split_MB: 215.5 +rank7 [2026-10-28 22:11:47,343] INFO : Step 15400/50000 rank7 | Train Loss: 1.5513 | Train Loss Unweighted: 0.7875 | Train Loss Components: Auxiliary: 0.3130 | Cardinality: 0.1800 | Classification: 0.1300 | Classification_binary: 0.2766 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1862 | Regression_eta: 0.0028 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2967 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0829 | Regression_eta: 0.0957 | Regression_pt: 0.0873 | Regression_sin_phi: 0.1016 | LR: 8.05e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1756s | Phase Times: max_pre_step_barrier: 0.1101 | max_batch_to_device: 0.0066 | max_forward: 0.7572 | max_loss: 0.4228 | max_backward: 1.7150 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6366.4 | uss_MB: 5741.6 | child_rss_MB: 5321.0 | allocated_MB: 346.3 | reserved_MB: 24612.0 | inactive_split_MB: 280.1 +rank7 [2026-10-28 22:16:06,324] INFO : Step 15500/50000 rank7 | Train Loss: 1.3262 | Train Loss Unweighted: 0.6688 | Train Loss Components: Auxiliary: 0.2642 | Cardinality: 0.1485 | Classification: 0.0986 | Classification_binary: 0.2128 | HitGrouping: 0.2171 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0861 | Regression_eta: 0.0879 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0766 | LR: 8.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9205s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6717 | max_loss: 0.2437 | max_backward: 1.4705 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.6680 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6368.1 | uss_MB: 5742.9 | child_rss_MB: 5326.7 | allocated_MB: 344.3 | reserved_MB: 25650.0 | inactive_split_MB: 195.5 +rank7 [2026-10-28 22:20:18,325] INFO : Step 15600/50000 rank7 | Train Loss: 1.2360 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1341 | Classification: 0.0982 | Classification_binary: 0.1880 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1702 | Regression_eta: 0.0021 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0883 | Regression_eta: 0.0494 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0690 | LR: 8.01e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.4189s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 2.4627 | max_loss: 0.7043 | max_backward: 2.6378 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.6129 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6373.1 | uss_MB: 5746.1 | child_rss_MB: 5328.0 | allocated_MB: 342.6 | reserved_MB: 25650.0 | inactive_split_MB: 190.9 +rank7 [2026-10-28 22:24:30,252] INFO : Step 15700/50000 rank7 | Train Loss: 1.4828 | Train Loss Unweighted: 0.7651 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.1701 | Classification: 0.1355 | Classification_binary: 0.2582 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1828 | Regression_eta: 0.0022 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0706 | Regression_eta: 0.0559 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0916 | LR: 7.98e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.9300s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0071 | max_forward: 1.0013 | max_loss: 0.3559 | max_backward: 2.9812 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8804 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6374.2 | uss_MB: 5747.2 | child_rss_MB: 5330.6 | allocated_MB: 348.9 | reserved_MB: 25650.0 | inactive_split_MB: 205.1 +rank7 [2026-10-28 22:28:46,454] INFO : Step 15800/50000 rank7 | Train Loss: 1.4507 | Train Loss Unweighted: 0.7629 | Train Loss Components: Auxiliary: 0.2990 | Cardinality: 0.1542 | Classification: 0.1164 | Classification_binary: 0.2507 | HitGrouping: 0.2156 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1943 | Regression_eta: 0.0023 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2599 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0805 | Regression_eta: 0.0826 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0948 | LR: 7.96e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3027s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0488 | max_forward: 0.7734 | max_loss: 0.1576 | max_backward: 1.6154 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.7873 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6376.3 | uss_MB: 5747.4 | child_rss_MB: 5329.9 | allocated_MB: 343.3 | reserved_MB: 25650.0 | inactive_split_MB: 193.3 +rank7 [2026-10-28 22:32:57,712] INFO : Step 15900/50000 rank7 | Train Loss: 1.2290 | Train Loss Unweighted: 0.6329 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1300 | Classification: 0.1008 | Classification_binary: 0.1914 | HitGrouping: 0.2128 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0661 | Regression_eta: 0.0500 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0746 | LR: 7.94e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3931s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0060 | max_forward: 0.8553 | max_loss: 0.1871 | max_backward: 1.6922 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6378.1 | uss_MB: 5737.7 | child_rss_MB: 5322.0 | allocated_MB: 341.2 | reserved_MB: 25650.0 | inactive_split_MB: 218.2 +rank7 [2026-10-28 22:37:10,742] INFO : Step 16000/50000 rank7 | Train Loss: 1.4158 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1648 | Classification: 0.1217 | Classification_binary: 0.2373 | HitGrouping: 0.2208 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1772 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.2450 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0826 | Regression_eta: 0.1012 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1074 | LR: 7.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0611s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6730 | max_loss: 0.1726 | max_backward: 1.4671 | max_optimizer_step: 0.1971 | max_tensorboard_logging: 0.7878 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6379.9 | uss_MB: 5752.4 | child_rss_MB: 5324.1 | allocated_MB: 345.1 | reserved_MB: 25650.0 | inactive_split_MB: 205.2 +rank7 [2026-10-28 22:37:11,681] INFO : Running validation on rank7 +rank7 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 22:37:11,681] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 22:38:03,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:38:04,032] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 22:38:30,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 22:38:31,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 22:38:32,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 22:38:33,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 22:38:43,278] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 22:38:43,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:38:44,323] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 22:39:00,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 22:39:02,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 22:39:03,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 22:39:05,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 22:39:15,458] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-28 22:39:15,560] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:39:16,146] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 22:39:31,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 22:39:32,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 22:39:32,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 22:39:33,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_16000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 22:39:43,436] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 22:44:03,866] INFO : Step 16100/50000 rank7 | Train Loss: 1.4898 | Train Loss Unweighted: 0.7352 | Train Loss Components: Auxiliary: 0.2980 | Cardinality: 0.1749 | Classification: 0.1104 | Classification_binary: 0.2584 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1803 | Regression_eta: 0.0026 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2839 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0881 | Regression_eta: 0.0879 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0930 | LR: 7.89e-04 | DataLoad Time: 0.0117s | Model Forward Time: 4.7748s | Phase Times: max_pre_step_barrier: 0.0420 | max_batch_to_device: 0.0278 | max_forward: 1.1120 | max_loss: 0.4532 | max_backward: 1.7233 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8150 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6496.2 | uss_MB: 5863.0 | child_rss_MB: 5316.2 | allocated_MB: 344.4 | reserved_MB: 23178.0 | inactive_split_MB: 291.7 +rank7 [2026-10-28 22:48:09,285] INFO : Step 16200/50000 rank7 | Train Loss: 1.3918 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2823 | Cardinality: 0.1924 | Classification: 0.1027 | Classification_binary: 0.2006 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1908 | Regression_eta: 0.0024 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0930 | Regression_energy: 0.1074 | Regression_eta: 0.0689 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0758 | LR: 7.86e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.6644s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.9065 | max_loss: 0.2576 | max_backward: 1.2704 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6494.2 | uss_MB: 5865.8 | child_rss_MB: 5326.3 | allocated_MB: 341.8 | reserved_MB: 25238.0 | inactive_split_MB: 213.5 +rank7 [2026-10-28 22:52:16,252] INFO : Step 16300/50000 rank7 | Train Loss: 1.4214 | Train Loss Unweighted: 0.7256 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1535 | Classification: 0.1255 | Classification_binary: 0.2234 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1845 | Regression_eta: 0.0024 | Regression_pt: 0.1873 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0785 | Regression_eta: 0.0826 | Regression_pt: 0.0840 | Regression_sin_phi: 0.1095 | LR: 7.84e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1361s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0577 | max_forward: 0.8077 | max_loss: 0.1474 | max_backward: 1.3555 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8504 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6493.5 | uss_MB: 5865.4 | child_rss_MB: 5334.5 | allocated_MB: 342.9 | reserved_MB: 25238.0 | inactive_split_MB: 243.1 +rank7 [2026-10-28 22:56:15,609] INFO : Step 16400/50000 rank7 | Train Loss: 1.3542 | Train Loss Unweighted: 0.6893 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1423 | Classification: 0.1043 | Classification_binary: 0.2537 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1625 | Regression_eta: 0.0022 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2583 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0738 | Regression_eta: 0.0671 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0784 | LR: 7.81e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.4175s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.8437 | max_loss: 0.1523 | max_backward: 1.7416 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.6893 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0092 | Memory: rss_MB: 6495.9 | uss_MB: 4859.9 | child_rss_MB: 5342.6 | allocated_MB: 343.5 | reserved_MB: 25238.0 | inactive_split_MB: 290.5 +rank7 [2026-10-28 23:00:23,724] INFO : Step 16500/50000 rank7 | Train Loss: 1.3911 | Train Loss Unweighted: 0.6567 | Train Loss Components: Auxiliary: 0.2847 | Cardinality: 0.2147 | Classification: 0.0982 | Classification_binary: 0.2034 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1764 | Regression_eta: 0.0023 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0948 | Regression_eta: 0.0731 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0831 | LR: 7.79e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9992s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0074 | max_forward: 0.6927 | max_loss: 0.2546 | max_backward: 1.3823 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.7641 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6494.7 | uss_MB: 5870.1 | child_rss_MB: 5327.9 | allocated_MB: 342.4 | reserved_MB: 25238.0 | inactive_split_MB: 260.1 +rank7 [2026-10-28 23:04:26,322] INFO : Step 16600/50000 rank7 | Train Loss: 1.2757 | Train Loss Unweighted: 0.6693 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1417 | Classification: 0.1229 | Classification_binary: 0.1953 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1740 | Regression_eta: 0.0019 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2212 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0891 | Regression_eta: 0.0884 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0663 | LR: 7.77e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.3637s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 0.8930 | max_loss: 0.1798 | max_backward: 1.6455 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6498.7 | uss_MB: 5832.6 | child_rss_MB: 5344.6 | allocated_MB: 343.8 | reserved_MB: 25238.0 | inactive_split_MB: 229.7 +rank7 [2026-10-28 23:08:31,108] INFO : Step 16700/50000 rank7 | Train Loss: 1.4448 | Train Loss Unweighted: 0.7323 | Train Loss Components: Auxiliary: 0.2948 | Cardinality: 0.1521 | Classification: 0.1200 | Classification_binary: 0.2524 | HitGrouping: 0.2120 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1769 | Regression_eta: 0.0025 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2613 | Regression_cos_phi: 0.0960 | Regression_energy: 0.0838 | Regression_eta: 0.0781 | Regression_pt: 0.0877 | Regression_sin_phi: 0.1155 | LR: 7.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0575s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7218 | max_loss: 0.1494 | max_backward: 1.5627 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5053 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6500.4 | uss_MB: 5874.4 | child_rss_MB: 5330.0 | allocated_MB: 345.6 | reserved_MB: 26192.0 | inactive_split_MB: 252.7 +rank7 [2026-10-28 23:12:28,974] INFO : Step 16800/50000 rank7 | Train Loss: 1.3775 | Train Loss Unweighted: 0.7025 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1637 | Classification: 0.1136 | Classification_binary: 0.2368 | HitGrouping: 0.2121 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1737 | Regression_eta: 0.0022 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2367 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0716 | Regression_eta: 0.0664 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0680 | LR: 7.72e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.0666s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7504 | max_loss: 0.2306 | max_backward: 1.6658 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5045 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6500.9 | uss_MB: 5874.9 | child_rss_MB: 5343.9 | allocated_MB: 346.0 | reserved_MB: 26192.0 | inactive_split_MB: 254.0 +rank7 [2026-10-28 23:16:36,875] INFO : Step 16900/50000 rank7 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6463 | Train Loss Components: Auxiliary: 0.2542 | Cardinality: 0.1430 | Classification: 0.1091 | Classification_binary: 0.1752 | HitGrouping: 0.2111 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0925 | Regression_eta: 0.0602 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0885 | LR: 7.69e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.7655s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0062 | max_forward: 0.6223 | max_loss: 0.1507 | max_backward: 1.3072 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.6935 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6493.1 | uss_MB: 5875.4 | child_rss_MB: 5344.2 | allocated_MB: 343.9 | reserved_MB: 26192.0 | inactive_split_MB: 276.7 +rank7 [2026-10-28 23:20:36,992] INFO : Step 17000/50000 rank7 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6931 | Train Loss Components: Auxiliary: 0.2787 | Cardinality: 0.1408 | Classification: 0.1200 | Classification_binary: 0.2247 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1712 | Regression_eta: 0.0020 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2256 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0780 | Regression_eta: 0.0788 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0959 | LR: 7.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9355s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6728 | max_loss: 0.1447 | max_backward: 1.4545 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6497.6 | uss_MB: 5879.3 | child_rss_MB: 5341.7 | allocated_MB: 342.6 | reserved_MB: 26192.0 | inactive_split_MB: 276.6 +rank7 [2026-10-28 23:20:37,935] INFO : Running validation on rank7 +rank7 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:20:37,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 23:21:28,542] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:21:29,100] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 23:21:55,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 23:21:57,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 23:22:00,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 23:22:02,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 23:22:12,122] INFO : Time taken to make predictions on device 7 is: 0.32 min +rank7 [2026-10-28 23:22:12,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:22:12,960] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 23:22:28,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 23:22:29,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 23:22:30,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 23:22:31,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 23:22:41,826] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:22:42,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:22:42,878] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 23:22:58,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 23:22:59,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 23:22:59,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 23:23:00,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_17000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 23:23:10,717] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 23:27:38,314] INFO : Step 17100/50000 rank7 | Train Loss: 1.3620 | Train Loss Unweighted: 0.7052 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1564 | Classification: 0.1116 | Classification_binary: 0.2457 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1716 | Regression_eta: 0.0021 | Regression_pt: 0.1722 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2149 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0778 | Regression_eta: 0.0527 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0653 | LR: 7.64e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.1659s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8653 | max_loss: 0.1831 | max_backward: 1.9822 | max_optimizer_step: 0.2269 | max_tensorboard_logging: 1.1231 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 6588.0 | uss_MB: 4137.2 | child_rss_MB: 5341.2 | allocated_MB: 346.7 | reserved_MB: 22812.0 | inactive_split_MB: 261.3 +rank7 [2026-10-28 23:31:38,671] INFO : Step 17200/50000 rank7 | Train Loss: 1.2977 | Train Loss Unweighted: 0.6379 | Train Loss Components: Auxiliary: 0.2565 | Cardinality: 0.1592 | Classification: 0.1006 | Classification_binary: 0.2024 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1670 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0803 | Regression_eta: 0.0600 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0848 | LR: 7.62e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1944s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.7872 | max_loss: 0.1645 | max_backward: 1.3999 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7667 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6585.9 | uss_MB: 5970.5 | child_rss_MB: 5317.2 | allocated_MB: 344.3 | reserved_MB: 24820.0 | inactive_split_MB: 199.7 +rank7 [2026-10-28 23:35:49,549] INFO : Step 17300/50000 rank7 | Train Loss: 1.2647 | Train Loss Unweighted: 0.6425 | Train Loss Components: Auxiliary: 0.2597 | Cardinality: 0.1344 | Classification: 0.1009 | Classification_binary: 0.2017 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1672 | Regression_eta: 0.0022 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2040 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0706 | Regression_eta: 0.0795 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0898 | LR: 7.59e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.5265s | Phase Times: max_pre_step_barrier: 0.0063 | max_batch_to_device: 0.0064 | max_forward: 0.9856 | max_loss: 0.3669 | max_backward: 2.6439 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9993 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.1347 | Memory: rss_MB: 6583.9 | uss_MB: 5728.8 | child_rss_MB: 5346.2 | allocated_MB: 345.5 | reserved_MB: 24820.0 | inactive_split_MB: 232.5 +rank7 [2026-10-28 23:39:53,702] INFO : Step 17400/50000 rank7 | Train Loss: 1.4141 | Train Loss Unweighted: 0.7186 | Train Loss Components: Auxiliary: 0.3006 | Cardinality: 0.1644 | Classification: 0.1085 | Classification_binary: 0.2578 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1735 | Regression_eta: 0.0023 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2356 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0919 | Regression_eta: 0.0907 | Regression_pt: 0.0956 | Regression_sin_phi: 0.1284 | LR: 7.57e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.3527s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8039 | max_loss: 0.2146 | max_backward: 1.6449 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 2.7771 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0911 | Memory: rss_MB: 6585.5 | uss_MB: 5972.4 | child_rss_MB: 5341.1 | allocated_MB: 350.6 | reserved_MB: 24820.0 | inactive_split_MB: 288.7 +rank7 [2026-10-28 23:43:53,975] INFO : Step 17500/50000 rank7 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6510 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1605 | Classification: 0.0872 | Classification_binary: 0.2143 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1734 | Regression_eta: 0.0019 | Regression_pt: 0.1721 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0835 | Regression_eta: 0.0630 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0729 | LR: 7.54e-04 | DataLoad Time: 0.0222s | Model Forward Time: 3.9626s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6991 | max_loss: 0.1488 | max_backward: 1.3497 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7188 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0977 | Memory: rss_MB: 6587.0 | uss_MB: 5844.6 | child_rss_MB: 5344.4 | allocated_MB: 342.7 | reserved_MB: 25682.0 | inactive_split_MB: 174.5 +rank7 [2026-10-28 23:48:08,874] INFO : Step 17600/50000 rank7 | Train Loss: 1.1600 | Train Loss Unweighted: 0.5867 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1251 | Classification: 0.0954 | Classification_binary: 0.1793 | HitGrouping: 0.2088 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1533 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0662 | Regression_eta: 0.0540 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0671 | LR: 7.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5561s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0445 | max_forward: 0.6445 | max_loss: 0.1399 | max_backward: 1.2133 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5441 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6586.0 | uss_MB: 5971.8 | child_rss_MB: 5323.1 | allocated_MB: 344.2 | reserved_MB: 25682.0 | inactive_split_MB: 213.7 +rank7 [2026-10-28 23:52:17,699] INFO : Step 17700/50000 rank7 | Train Loss: 1.3405 | Train Loss Unweighted: 0.6682 | Train Loss Components: Auxiliary: 0.2842 | Cardinality: 0.1722 | Classification: 0.1054 | Classification_binary: 0.2183 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1708 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0793 | Regression_eta: 0.0941 | Regression_pt: 0.0824 | Regression_sin_phi: 0.1179 | LR: 7.49e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.8655s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0065 | max_forward: 0.7144 | max_loss: 0.2491 | max_backward: 1.5081 | max_optimizer_step: 0.2045 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6585.0 | uss_MB: 5976.0 | child_rss_MB: 5346.5 | allocated_MB: 343.3 | reserved_MB: 25684.0 | inactive_split_MB: 192.2 +rank7 [2026-10-28 23:56:40,919] INFO : Step 17800/50000 rank7 | Train Loss: 1.3770 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2806 | Cardinality: 0.1613 | Classification: 0.1058 | Classification_binary: 0.2345 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1679 | Regression_eta: 0.0024 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2350 | Regression_cos_phi: 0.1203 | Regression_energy: 0.0723 | Regression_eta: 0.0851 | Regression_pt: 0.0748 | Regression_sin_phi: 0.1081 | LR: 7.47e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1266s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7705 | max_loss: 0.5735 | max_backward: 2.6855 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.4958 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6567.9 | uss_MB: 5976.6 | child_rss_MB: 5248.0 | allocated_MB: 343.9 | reserved_MB: 25684.0 | inactive_split_MB: 230.9 +rank7 [2026-10-29 00:00:47,130] INFO : Step 17900/50000 rank7 | Train Loss: 1.1914 | Train Loss Unweighted: 0.6026 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1393 | Classification: 0.0929 | Classification_binary: 0.1720 | HitGrouping: 0.2107 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0808 | Regression_eta: 0.0473 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0560 | LR: 7.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2725s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6203 | max_loss: 0.2390 | max_backward: 1.5377 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.7450 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 6569.9 | uss_MB: 5881.0 | child_rss_MB: 5280.9 | allocated_MB: 340.2 | reserved_MB: 26706.0 | inactive_split_MB: 248.9 +rank7 [2026-10-29 00:04:53,689] INFO : Step 18000/50000 rank7 | Train Loss: 1.3368 | Train Loss Unweighted: 0.6691 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1758 | Classification: 0.1113 | Classification_binary: 0.2113 | HitGrouping: 0.2118 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1700 | Regression_eta: 0.0022 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0693 | Regression_eta: 0.0740 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0908 | LR: 7.42e-04 | DataLoad Time: 0.0167s | Model Forward Time: 13.5872s | Phase Times: max_pre_step_barrier: 0.0736 | max_batch_to_device: 0.0252 | max_forward: 3.9392 | max_loss: 1.1226 | max_backward: 10.7533 | max_optimizer_step: 0.3088 | max_tensorboard_logging: 0.5909 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6570.7 | uss_MB: 5977.3 | child_rss_MB: 5277.4 | allocated_MB: 342.1 | reserved_MB: 26706.0 | inactive_split_MB: 168.4 +rank7 [2026-10-29 00:04:54,634] INFO : Running validation on rank7 +rank7 [2026-10-29 00:04:54,635] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 00:04:54,635] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:05:43,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:05:43,820] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 00:06:10,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:06:11,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:06:12,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:06:13,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:06:23,942] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 00:06:24,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:06:25,297] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 00:06:41,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:06:42,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:06:43,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:06:44,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:06:54,445] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 00:06:54,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:06:55,431] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 00:07:11,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:07:12,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:07:12,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:07:13,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_18000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:07:23,259] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 00:11:38,772] INFO : Step 18100/50000 rank7 | Train Loss: 1.5331 | Train Loss Unweighted: 0.7165 | Train Loss Components: Auxiliary: 0.3077 | Cardinality: 0.2738 | Classification: 0.1261 | Classification_binary: 0.2459 | HitGrouping: 0.2115 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2927 | Regression_cos_phi: 0.1299 | Regression_energy: 0.0970 | Regression_eta: 0.0963 | Regression_pt: 0.1006 | Regression_sin_phi: 0.1054 | LR: 7.39e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.2930s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0062 | max_forward: 0.9205 | max_loss: 0.4635 | max_backward: 1.6576 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.4944 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0748 | Memory: rss_MB: 8101.6 | uss_MB: 7397.2 | child_rss_MB: 5241.0 | allocated_MB: 346.4 | reserved_MB: 23812.0 | inactive_split_MB: 277.3 +rank7 [2026-10-29 00:15:47,602] INFO : Step 18200/50000 rank7 | Train Loss: 1.3411 | Train Loss Unweighted: 0.6776 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1663 | Classification: 0.1017 | Classification_binary: 0.2095 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1809 | Regression_eta: 0.0019 | Regression_pt: 0.1815 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2407 | Regression_cos_phi: 0.1049 | Regression_energy: 0.1046 | Regression_eta: 0.0641 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0839 | LR: 7.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1154s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0460 | max_forward: 0.7429 | max_loss: 0.1491 | max_backward: 1.5264 | max_optimizer_step: 0.2786 | max_tensorboard_logging: 0.5957 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6634.4 | uss_MB: 6042.8 | child_rss_MB: 5244.2 | allocated_MB: 344.4 | reserved_MB: 23814.0 | inactive_split_MB: 200.1 +rank7 [2026-10-29 00:19:54,087] INFO : Step 18300/50000 rank7 | Train Loss: 1.2735 | Train Loss Unweighted: 0.6284 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1528 | Classification: 0.1060 | Classification_binary: 0.1908 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1623 | Regression_eta: 0.0020 | Regression_pt: 0.1651 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0848 | Regression_eta: 0.0561 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0805 | LR: 7.34e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8060s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.6459 | max_loss: 0.4136 | max_backward: 1.3456 | max_optimizer_step: 0.2902 | max_tensorboard_logging: 0.5978 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6639.6 | uss_MB: 6045.3 | child_rss_MB: 5252.4 | allocated_MB: 346.4 | reserved_MB: 23814.0 | inactive_split_MB: 225.7 +rank7 [2026-10-29 00:24:07,390] INFO : Step 18400/50000 rank7 | Train Loss: 1.3888 | Train Loss Unweighted: 0.6759 | Train Loss Components: Auxiliary: 0.2832 | Cardinality: 0.1872 | Classification: 0.1175 | Classification_binary: 0.2379 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1580 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2492 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0770 | Regression_eta: 0.1069 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0943 | LR: 7.31e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4811s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7584 | max_loss: 0.2611 | max_backward: 1.6883 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9367 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6637.2 | uss_MB: 6046.1 | child_rss_MB: 5243.1 | allocated_MB: 346.0 | reserved_MB: 23814.0 | inactive_split_MB: 196.0 +rank7 [2026-10-29 00:27:44,479] INFO : Step 18500/50000 rank7 | Train Loss: 1.3229 | Train Loss Unweighted: 0.6649 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1383 | Classification: 0.1064 | Classification_binary: 0.2201 | HitGrouping: 0.2117 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1653 | Regression_eta: 0.0022 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2341 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0787 | Regression_eta: 0.0552 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0768 | LR: 7.29e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5464s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0063 | max_forward: 0.7953 | max_loss: 0.1540 | max_backward: 1.8095 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.8568 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6641.4 | uss_MB: 6048.5 | child_rss_MB: 5227.8 | allocated_MB: 344.9 | reserved_MB: 24820.0 | inactive_split_MB: 281.4 +rank7 [2026-10-29 00:31:24,471] INFO : Step 18600/50000 rank7 | Train Loss: 1.2737 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2618 | Cardinality: 0.1513 | Classification: 0.0906 | Classification_binary: 0.1832 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1778 | Regression_eta: 0.0020 | Regression_pt: 0.1820 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0754 | Regression_eta: 0.0641 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0726 | LR: 7.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1893s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.6385 | max_loss: 0.2465 | max_backward: 1.5588 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.8580 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6641.5 | uss_MB: 6047.2 | child_rss_MB: 5253.2 | allocated_MB: 342.9 | reserved_MB: 24820.0 | inactive_split_MB: 217.1 +rank7 [2026-10-29 00:35:06,907] INFO : Step 18700/50000 rank7 | Train Loss: 1.2715 | Train Loss Unweighted: 0.6508 | Train Loss Components: Auxiliary: 0.2640 | Cardinality: 0.1357 | Classification: 0.1146 | Classification_binary: 0.2109 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0719 | Regression_eta: 0.0461 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0655 | LR: 7.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.9498s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6659 | max_loss: 0.1512 | max_backward: 1.4189 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 2.7373 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6643.3 | uss_MB: 6048.6 | child_rss_MB: 5252.2 | allocated_MB: 347.9 | reserved_MB: 24820.0 | inactive_split_MB: 174.7 +rank7 [2026-10-29 00:38:49,579] INFO : Step 18800/50000 rank7 | Train Loss: 1.3396 | Train Loss Unweighted: 0.6764 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1825 | Classification: 0.1038 | Classification_binary: 0.2223 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1730 | Regression_eta: 0.0022 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.2255 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0914 | Regression_eta: 0.0601 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0746 | LR: 7.21e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4358s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8361 | max_loss: 0.1560 | max_backward: 1.7421 | max_optimizer_step: 0.2287 | max_tensorboard_logging: 0.7392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6650.3 | uss_MB: 6052.5 | child_rss_MB: 5245.6 | allocated_MB: 345.1 | reserved_MB: 25590.0 | inactive_split_MB: 267.1 +rank7 [2026-10-29 00:42:28,238] INFO : Step 18900/50000 rank7 | Train Loss: 1.1780 | Train Loss Unweighted: 0.5860 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1245 | Classification: 0.0945 | Classification_binary: 0.1787 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1525 | Regression_eta: 0.0020 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0736 | Regression_eta: 0.0574 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0487 | LR: 7.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0122s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7053 | max_loss: 0.1414 | max_backward: 1.3546 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.7681 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0478 | Memory: rss_MB: 6645.7 | uss_MB: 6049.6 | child_rss_MB: 5242.8 | allocated_MB: 345.2 | reserved_MB: 25830.0 | inactive_split_MB: 205.8 +rank7 [2026-10-29 00:46:10,109] INFO : Step 19000/50000 rank7 | Train Loss: 1.3888 | Train Loss Unweighted: 0.7071 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1730 | Classification: 0.1215 | Classification_binary: 0.2040 | HitGrouping: 0.2172 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0022 | Regression_pt: 0.1892 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0899 | Regression_eta: 0.0677 | Regression_pt: 0.0950 | Regression_sin_phi: 0.1059 | LR: 7.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1527s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6790 | max_loss: 0.2519 | max_backward: 1.4935 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1115 | Memory: rss_MB: 6645.1 | uss_MB: 6048.4 | child_rss_MB: 5243.4 | allocated_MB: 343.7 | reserved_MB: 25830.0 | inactive_split_MB: 236.1 +rank7 [2026-10-29 00:46:11,054] INFO : Running validation on rank7 +rank7 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 00:46:11,054] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:46:58,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:46:59,173] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 00:47:25,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:47:25,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:47:26,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:47:27,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:47:37,952] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:47:38,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:47:38,899] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 00:47:54,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:47:55,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:47:56,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:47:57,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:48:07,643] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:48:08,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:48:09,174] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 00:48:24,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:48:24,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:48:25,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:48:26,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_19000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:48:36,068] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 00:52:41,642] INFO : Step 19100/50000 rank7 | Train Loss: 1.4153 | Train Loss Unweighted: 0.7129 | Train Loss Components: Auxiliary: 0.2950 | Cardinality: 0.1930 | Classification: 0.1193 | Classification_binary: 0.2377 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1749 | Regression_eta: 0.0022 | Regression_pt: 0.1767 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.2654 | Regression_cos_phi: 0.1123 | Regression_energy: 0.0976 | Regression_eta: 0.0779 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0895 | LR: 7.13e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5903s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0887 | max_forward: 0.8602 | max_loss: 0.4256 | max_backward: 1.8019 | max_optimizer_step: 0.2257 | max_tensorboard_logging: 0.7163 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0883 | Memory: rss_MB: 8372.9 | uss_MB: 7777.7 | child_rss_MB: 5240.0 | allocated_MB: 343.0 | reserved_MB: 21856.0 | inactive_split_MB: 275.0 +rank7 [2026-10-29 00:56:31,489] INFO : Step 19200/50000 rank7 | Train Loss: 1.2052 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1475 | Classification: 0.0918 | Classification_binary: 0.1941 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1565 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0740 | Regression_eta: 0.0468 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0628 | LR: 7.10e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.7734s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.6633 | max_loss: 0.2460 | max_backward: 1.2259 | max_optimizer_step: 0.2979 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6774.2 | uss_MB: 6178.9 | child_rss_MB: 5244.7 | allocated_MB: 342.5 | reserved_MB: 23782.0 | inactive_split_MB: 238.2 +rank7 [2026-10-29 01:00:15,643] INFO : Step 19300/50000 rank7 | Train Loss: 1.2113 | Train Loss Unweighted: 0.6286 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1810 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1691 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0738 | Regression_eta: 0.0586 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0671 | LR: 7.08e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6454s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6069 | max_loss: 0.1478 | max_backward: 1.3108 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5569 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6776.5 | uss_MB: 6181.6 | child_rss_MB: 5240.7 | allocated_MB: 341.6 | reserved_MB: 23782.0 | inactive_split_MB: 233.6 +rank7 [2026-10-29 01:03:57,849] INFO : Step 19400/50000 rank7 | Train Loss: 1.5078 | Train Loss Unweighted: 0.7418 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.2064 | Classification: 0.1216 | Classification_binary: 0.2369 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1885 | Regression_eta: 0.0024 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2475 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0894 | Regression_eta: 0.0879 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0895 | LR: 7.05e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0883 | max_forward: 0.8010 | max_loss: 0.1497 | max_backward: 1.5021 | max_optimizer_step: 0.2577 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6778.1 | uss_MB: 5340.4 | child_rss_MB: 5248.6 | allocated_MB: 346.2 | reserved_MB: 23782.0 | inactive_split_MB: 179.5 +rank7 [2026-10-29 01:07:42,206] INFO : Step 19500/50000 rank7 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6851 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1421 | Classification: 0.0986 | Classification_binary: 0.2285 | HitGrouping: 0.2140 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2208 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0928 | Regression_eta: 0.0576 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0598 | LR: 7.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0508s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0110 | max_forward: 0.7458 | max_loss: 0.1513 | max_backward: 1.4598 | max_optimizer_step: 0.2763 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6774.1 | uss_MB: 6182.3 | child_rss_MB: 5222.4 | allocated_MB: 343.9 | reserved_MB: 24232.0 | inactive_split_MB: 278.6 +rank7 [2026-10-29 01:11:25,842] INFO : Step 19600/50000 rank7 | Train Loss: 1.2245 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2571 | Cardinality: 0.1431 | Classification: 0.0892 | Classification_binary: 0.1828 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0018 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0552 | Regression_energy: 0.1051 | Regression_eta: 0.0576 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0473 | LR: 7.00e-04 | DataLoad Time: 0.0229s | Model Forward Time: 3.9367s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6394 | max_loss: 0.2477 | max_backward: 1.3940 | max_optimizer_step: 0.2549 | max_tensorboard_logging: 0.6126 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6779.4 | uss_MB: 6183.8 | child_rss_MB: 5236.3 | allocated_MB: 340.8 | reserved_MB: 24232.0 | inactive_split_MB: 168.8 +rank7 [2026-10-29 01:15:28,502] INFO : Step 19700/50000 rank7 | Train Loss: 1.3157 | Train Loss Unweighted: 0.6791 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1360 | Classification: 0.1241 | Classification_binary: 0.2084 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1707 | Regression_eta: 0.0021 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0823 | Regression_eta: 0.1003 | Regression_pt: 0.0858 | Regression_sin_phi: 0.1152 | LR: 6.97e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.3551s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.6883 | max_loss: 0.2513 | max_backward: 3.0681 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0853 | Memory: rss_MB: 6778.4 | uss_MB: 5953.6 | child_rss_MB: 5244.0 | allocated_MB: 348.5 | reserved_MB: 24358.0 | inactive_split_MB: 457.7 +rank7 [2026-10-29 01:19:24,798] INFO : Step 19800/50000 rank7 | Train Loss: 1.4118 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.1769 | Classification: 0.1075 | Classification_binary: 0.2457 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1884 | Regression_eta: 0.0020 | Regression_pt: 0.1896 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2428 | Regression_cos_phi: 0.0664 | Regression_energy: 0.1160 | Regression_eta: 0.0494 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0862 | LR: 6.94e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3400s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0656 | max_forward: 0.7985 | max_loss: 0.1492 | max_backward: 1.7787 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5856 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6783.1 | uss_MB: 6189.8 | child_rss_MB: 5243.3 | allocated_MB: 347.1 | reserved_MB: 25242.0 | inactive_split_MB: 286.0 +rank7 [2026-10-29 01:23:26,313] INFO : Step 19900/50000 rank7 | Train Loss: 1.2868 | Train Loss Unweighted: 0.6569 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1448 | Classification: 0.1067 | Classification_binary: 0.1948 | HitGrouping: 0.2119 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1755 | Regression_eta: 0.0021 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0803 | Regression_eta: 0.0619 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0563 | LR: 6.92e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.7472s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.6959 | max_loss: 0.1442 | max_backward: 1.3168 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.5319 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6781.7 | uss_MB: 6187.8 | child_rss_MB: 5221.7 | allocated_MB: 344.2 | reserved_MB: 25242.0 | inactive_split_MB: 214.1 +rank7 [2026-10-29 01:27:15,700] INFO : Step 20000/50000 rank7 | Train Loss: 1.3131 | Train Loss Unweighted: 0.6575 | Train Loss Components: Auxiliary: 0.2712 | Cardinality: 0.1369 | Classification: 0.1102 | Classification_binary: 0.1903 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1745 | Regression_eta: 0.0024 | Regression_pt: 0.1778 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0688 | Regression_eta: 0.0932 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0684 | LR: 6.89e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8755s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0069 | max_forward: 0.6301 | max_loss: 0.2293 | max_backward: 1.2752 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.6236 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6784.2 | uss_MB: 5913.0 | child_rss_MB: 5245.1 | allocated_MB: 346.5 | reserved_MB: 25242.0 | inactive_split_MB: 235.1 +rank7 [2026-10-29 01:27:16,848] INFO : Running validation on rank7 +rank7 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 01:27:16,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 01:28:06,612] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:28:07,178] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 01:28:33,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 01:28:34,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 01:28:35,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 01:28:36,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 01:28:46,978] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:28:48,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:28:48,586] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 01:29:04,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 01:29:05,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 01:29:06,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 01:29:07,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 01:29:17,468] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 01:29:18,019] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:29:18,549] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 01:29:33,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 01:29:34,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 01:29:34,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 01:29:35,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_20000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 01:29:45,461] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 01:33:54,476] INFO : Step 20100/50000 rank7 | Train Loss: 1.4817 | Train Loss Unweighted: 0.7566 | Train Loss Components: Auxiliary: 0.3135 | Cardinality: 0.1801 | Classification: 0.1128 | Classification_binary: 0.2574 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1896 | Regression_eta: 0.0022 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.2655 | Regression_cos_phi: 0.1218 | Regression_energy: 0.1131 | Regression_eta: 0.0861 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0768 | LR: 6.86e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.4730s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 1.1202 | max_loss: 0.3361 | max_backward: 2.5911 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.5458 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 8600.4 | uss_MB: 7998.9 | child_rss_MB: 5282.4 | allocated_MB: 345.7 | reserved_MB: 23960.0 | inactive_split_MB: 258.3 +rank7 [2026-10-29 01:37:52,759] INFO : Step 20200/50000 rank7 | Train Loss: 1.2591 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1327 | Classification: 0.0910 | Classification_binary: 0.2056 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1668 | Regression_eta: 0.0020 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2245 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0963 | Regression_eta: 0.0650 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0820 | LR: 6.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9220s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6789 | max_loss: 0.1556 | max_backward: 1.2288 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.6545 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6865.1 | uss_MB: 6264.9 | child_rss_MB: 5280.5 | allocated_MB: 343.4 | reserved_MB: 24202.0 | inactive_split_MB: 179.6 +rank7 [2026-10-29 01:41:55,630] INFO : Step 20300/50000 rank7 | Train Loss: 1.2223 | Train Loss Unweighted: 0.6082 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1481 | Classification: 0.0985 | Classification_binary: 0.1755 | HitGrouping: 0.2178 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1643 | Regression_eta: 0.0017 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0877 | Regression_eta: 0.0574 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0768 | LR: 6.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6955s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6144 | max_loss: 0.4399 | max_backward: 1.3395 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 0.5456 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6862.7 | uss_MB: 6265.7 | child_rss_MB: 5254.5 | allocated_MB: 340.9 | reserved_MB: 25140.0 | inactive_split_MB: 224.5 +rank7 [2026-10-29 01:45:46,289] INFO : Step 20400/50000 rank7 | Train Loss: 1.3334 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2798 | Cardinality: 0.1633 | Classification: 0.1100 | Classification_binary: 0.2200 | HitGrouping: 0.2131 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0019 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0972 | Regression_eta: 0.0681 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0904 | LR: 6.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1292s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.6835 | max_loss: 0.5553 | max_backward: 2.5582 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.6005 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6863.1 | uss_MB: 6266.2 | child_rss_MB: 5247.9 | allocated_MB: 346.4 | reserved_MB: 25140.0 | inactive_split_MB: 177.4 +rank7 [2026-10-29 01:49:32,950] INFO : Step 20500/50000 rank7 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7081 | Train Loss Components: Auxiliary: 0.2908 | Cardinality: 0.1547 | Classification: 0.1127 | Classification_binary: 0.2506 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1697 | Regression_eta: 0.0022 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2446 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0994 | Regression_eta: 0.0658 | Regression_pt: 0.1035 | Regression_sin_phi: 0.0793 | LR: 6.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3415s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8645 | max_loss: 0.1537 | max_backward: 1.7247 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6869.1 | uss_MB: 6272.8 | child_rss_MB: 5275.8 | allocated_MB: 344.0 | reserved_MB: 25140.0 | inactive_split_MB: 265.4 +rank7 [2026-10-29 01:53:32,129] INFO : Step 20600/50000 rank7 | Train Loss: 1.3025 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2686 | Cardinality: 0.1589 | Classification: 0.0961 | Classification_binary: 0.1869 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1911 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0660 | Regression_energy: 0.1105 | Regression_eta: 0.0562 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0805 | LR: 6.73e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.0724s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.6582 | max_loss: 0.2510 | max_backward: 1.4168 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.7064 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6860.8 | uss_MB: 6168.9 | child_rss_MB: 5246.0 | allocated_MB: 343.6 | reserved_MB: 25140.0 | inactive_split_MB: 226.8 +rank7 [2026-10-29 01:57:29,941] INFO : Step 20700/50000 rank7 | Train Loss: 1.2842 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2676 | Cardinality: 0.1422 | Classification: 0.1178 | Classification_binary: 0.1965 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1630 | Regression_eta: 0.0021 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0935 | Regression_eta: 0.0496 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0703 | LR: 6.70e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.6629s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0071 | max_forward: 0.7041 | max_loss: 0.2172 | max_backward: 1.2044 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5982 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0594 | Memory: rss_MB: 6862.8 | uss_MB: 6270.0 | child_rss_MB: 5260.1 | allocated_MB: 345.3 | reserved_MB: 25976.0 | inactive_split_MB: 184.7 +rank7 [2026-10-29 02:01:35,425] INFO : Step 20800/50000 rank7 | Train Loss: 1.4280 | Train Loss Unweighted: 0.7397 | Train Loss Components: Auxiliary: 0.2956 | Cardinality: 0.1674 | Classification: 0.1140 | Classification_binary: 0.2502 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1851 | Regression_eta: 0.0021 | Regression_pt: 0.1861 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0909 | Regression_eta: 0.0608 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0747 | LR: 6.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8011s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 1.0918 | max_loss: 0.2437 | max_backward: 2.0987 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6865.4 | uss_MB: 6274.2 | child_rss_MB: 5255.1 | allocated_MB: 344.0 | reserved_MB: 25976.0 | inactive_split_MB: 246.7 +rank7 [2026-10-29 02:05:49,615] INFO : Step 20900/50000 rank7 | Train Loss: 1.2490 | Train Loss Unweighted: 0.6333 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1386 | Classification: 0.0917 | Classification_binary: 0.2010 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0808 | Regression_eta: 0.0479 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0799 | LR: 6.65e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.1298s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0160 | max_forward: 1.0731 | max_loss: 0.2339 | max_backward: 2.4848 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.6678 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1072 | Memory: rss_MB: 6862.3 | uss_MB: 5111.7 | child_rss_MB: 5271.4 | allocated_MB: 342.4 | reserved_MB: 25976.0 | inactive_split_MB: 175.9 +rank7 [2026-10-29 02:09:53,644] INFO : Step 21000/50000 rank7 | Train Loss: 1.3680 | Train Loss Unweighted: 0.7056 | Train Loss Components: Auxiliary: 0.2849 | Cardinality: 0.1521 | Classification: 0.1141 | Classification_binary: 0.2008 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1917 | Regression_eta: 0.0020 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0752 | Regression_energy: 0.1052 | Regression_eta: 0.0621 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0706 | LR: 6.62e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.8420s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6275 | max_loss: 0.2516 | max_backward: 1.3440 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6860.9 | uss_MB: 6273.7 | child_rss_MB: 5237.9 | allocated_MB: 344.7 | reserved_MB: 25976.0 | inactive_split_MB: 221.3 +rank7 [2026-10-29 02:09:54,611] INFO : Running validation on rank7 +rank7 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:09:54,611] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:10:45,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:10:45,901] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 02:11:12,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:11:12,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:11:14,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:11:15,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:11:25,078] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 02:11:25,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:11:26,307] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 02:11:42,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:11:44,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:11:45,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:11:46,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:11:56,870] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 02:11:57,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:11:57,776] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 02:12:13,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:12:13,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:12:14,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:12:15,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_21000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:12:25,208] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 02:16:43,090] INFO : Step 21100/50000 rank7 | Train Loss: 1.4905 | Train Loss Unweighted: 0.7392 | Train Loss Components: Auxiliary: 0.3038 | Cardinality: 0.2002 | Classification: 0.1227 | Classification_binary: 0.2388 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1838 | Regression_eta: 0.0024 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2577 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0855 | Regression_eta: 0.0803 | Regression_pt: 0.0920 | Regression_sin_phi: 0.1209 | LR: 6.59e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0065 | max_forward: 0.9001 | max_loss: 0.1990 | max_backward: 1.9658 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.5605 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 8584.6 | uss_MB: 6300.3 | child_rss_MB: 5241.5 | allocated_MB: 345.5 | reserved_MB: 25338.0 | inactive_split_MB: 207.2 +rank7 [2026-10-29 02:17:02,261] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank7 [2026-10-29 02:17:02,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 02:20:40,310] INFO : Step 21200/50000 rank7 | Train Loss: 1.2987 | Train Loss Unweighted: 0.6778 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1420 | Classification: 0.1019 | Classification_binary: 0.1839 | HitGrouping: 0.2307 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1917 | Regression_eta: 0.0019 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0933 | Regression_eta: 0.0766 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0956 | LR: 6.56e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1633s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6668 | max_loss: 0.1982 | max_backward: 1.6432 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.5420 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0872 | Memory: rss_MB: 6976.1 | uss_MB: 6381.6 | child_rss_MB: 5236.3 | allocated_MB: 345.9 | reserved_MB: 25338.0 | inactive_split_MB: 271.3 +rank7 [2026-10-29 02:24:40,426] INFO : Step 21300/50000 rank7 | Train Loss: 1.3756 | Train Loss Unweighted: 0.6777 | Train Loss Components: Auxiliary: 0.2822 | Cardinality: 0.1773 | Classification: 0.0990 | Classification_binary: 0.2185 | HitGrouping: 0.2186 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1769 | Regression_eta: 0.0020 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2275 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0885 | Regression_eta: 0.0572 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0789 | LR: 6.54e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1537s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0528 | max_forward: 0.7608 | max_loss: 0.1735 | max_backward: 1.5416 | max_optimizer_step: 0.2600 | max_tensorboard_logging: 0.6054 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6975.6 | uss_MB: 6380.6 | child_rss_MB: 5234.1 | allocated_MB: 344.3 | reserved_MB: 26044.0 | inactive_split_MB: 228.7 +rank7 [2026-10-29 02:28:35,145] INFO : Step 21400/50000 rank7 | Train Loss: 1.4917 | Train Loss Unweighted: 0.7358 | Train Loss Components: Auxiliary: 0.3138 | Cardinality: 0.1802 | Classification: 0.1135 | Classification_binary: 0.2604 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1784 | Regression_eta: 0.0023 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2586 | Regression_cos_phi: 0.1046 | Regression_energy: 0.0840 | Regression_eta: 0.0609 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0823 | LR: 6.51e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4145s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7979 | max_loss: 0.1504 | max_backward: 1.5280 | max_optimizer_step: 0.2437 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 6941.9 | uss_MB: 6114.7 | child_rss_MB: 5235.0 | allocated_MB: 345.8 | reserved_MB: 27040.0 | inactive_split_MB: 253.0 +rank7 [2026-10-29 02:32:39,450] INFO : Step 21500/50000 rank7 | Train Loss: 1.2624 | Train Loss Unweighted: 0.6550 | Train Loss Components: Auxiliary: 0.2650 | Cardinality: 0.1318 | Classification: 0.0951 | Classification_binary: 0.1986 | HitGrouping: 0.2203 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1791 | Regression_eta: 0.0020 | Regression_pt: 0.1782 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0956 | Regression_eta: 0.0574 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0893 | LR: 6.48e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7862s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6440 | max_loss: 0.1972 | max_backward: 1.0745 | max_optimizer_step: 0.2379 | max_tensorboard_logging: 0.7883 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6942.5 | uss_MB: 6349.2 | child_rss_MB: 5211.9 | allocated_MB: 340.5 | reserved_MB: 27040.0 | inactive_split_MB: 250.8 +rank7 [2026-10-29 02:36:35,118] INFO : Step 21600/50000 rank7 | Train Loss: 1.4231 | Train Loss Unweighted: 0.7309 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1519 | Classification: 0.1119 | Classification_binary: 0.2165 | HitGrouping: 0.2396 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1972 | Regression_eta: 0.0025 | Regression_pt: 0.2004 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0920 | Classification_binary: 0.2259 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0907 | Regression_eta: 0.0849 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0898 | LR: 6.45e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.8437s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0397 | max_forward: 0.7040 | max_loss: 0.1435 | max_backward: 1.3364 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 6948.5 | uss_MB: 6354.6 | child_rss_MB: 5221.6 | allocated_MB: 345.6 | reserved_MB: 27040.0 | inactive_split_MB: 287.3 +rank7 [2026-10-29 02:40:30,748] INFO : Step 21700/50000 rank7 | Train Loss: 1.5231 | Train Loss Unweighted: 0.7851 | Train Loss Components: Auxiliary: 0.3180 | Cardinality: 0.1742 | Classification: 0.1208 | Classification_binary: 0.2559 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2010 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2556 | Regression_cos_phi: 0.0724 | Regression_energy: 0.1048 | Regression_eta: 0.0753 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0927 | LR: 6.43e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5964s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0204 | max_forward: 1.6899 | max_loss: 0.1494 | max_backward: 1.9732 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6948.0 | uss_MB: 6355.1 | child_rss_MB: 5219.4 | allocated_MB: 345.0 | reserved_MB: 27040.0 | inactive_split_MB: 287.2 +rank7 [2026-10-29 02:44:20,671] INFO : Step 21800/50000 rank7 | Train Loss: 1.4682 | Train Loss Unweighted: 0.7620 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1608 | Classification: 0.1117 | Classification_binary: 0.2350 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2045 | Regression_eta: 0.0023 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2335 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0969 | Regression_eta: 0.0781 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0865 | LR: 6.40e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.1201s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0060 | max_forward: 0.9658 | max_loss: 0.2116 | max_backward: 1.8709 | max_optimizer_step: 0.2338 | max_tensorboard_logging: 1.1025 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6947.2 | uss_MB: 6353.7 | child_rss_MB: 5214.5 | allocated_MB: 341.8 | reserved_MB: 28040.0 | inactive_split_MB: 192.2 +rank7 [2026-10-29 02:48:12,872] INFO : Step 21900/50000 rank7 | Train Loss: 1.3268 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1429 | Classification: 0.1147 | Classification_binary: 0.1855 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1904 | Regression_eta: 0.0021 | Regression_pt: 0.1907 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0933 | Regression_eta: 0.0604 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0800 | LR: 6.37e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2325s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.6331 | max_loss: 0.1459 | max_backward: 1.3744 | max_optimizer_step: 0.2143 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6944.9 | uss_MB: 6355.2 | child_rss_MB: 5213.4 | allocated_MB: 342.5 | reserved_MB: 28042.0 | inactive_split_MB: 237.5 +rank7 [2026-10-29 02:52:08,910] INFO : Step 22000/50000 rank7 | Train Loss: 1.4853 | Train Loss Unweighted: 0.7623 | Train Loss Components: Auxiliary: 0.3039 | Cardinality: 0.1849 | Classification: 0.1339 | Classification_binary: 0.2421 | HitGrouping: 0.2295 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1902 | Regression_eta: 0.0022 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2421 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0803 | Regression_eta: 0.0665 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0765 | LR: 6.34e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1563s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7153 | max_loss: 0.1751 | max_backward: 1.5563 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.7311 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6944.1 | uss_MB: 6357.3 | child_rss_MB: 5222.5 | allocated_MB: 342.3 | reserved_MB: 28042.0 | inactive_split_MB: 175.1 +rank7 [2026-10-29 02:52:09,891] INFO : Running validation on rank7 +rank7 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:52:09,891] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:52:57,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:52:58,578] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 02:53:25,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:53:27,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:53:28,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:53:29,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:53:39,500] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 02:53:40,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:53:40,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 02:53:56,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:53:57,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:53:58,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:53:59,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:54:09,358] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 02:54:10,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:54:10,678] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 02:54:26,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:54:26,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:54:27,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:54:28,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_22000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:54:38,078] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 02:58:46,449] INFO : Step 22100/50000 rank7 | Train Loss: 1.6600 | Train Loss Unweighted: 0.8335 | Train Loss Components: Auxiliary: 0.3302 | Cardinality: 0.1851 | Classification: 0.1197 | Classification_binary: 0.2799 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2137 | Regression_eta: 0.0027 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2832 | Regression_cos_phi: 0.0822 | Regression_energy: 0.1040 | Regression_eta: 0.0817 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0992 | LR: 6.32e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4339s | Phase Times: max_pre_step_barrier: 0.0068 | max_batch_to_device: 0.0067 | max_forward: 0.8949 | max_loss: 0.1621 | max_backward: 1.5261 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.7045 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8820.7 | uss_MB: 8235.2 | child_rss_MB: 5222.5 | allocated_MB: 345.0 | reserved_MB: 24074.0 | inactive_split_MB: 271.1 +rank7 [2026-10-29 03:02:36,686] INFO : Step 22200/50000 rank7 | Train Loss: 1.3817 | Train Loss Unweighted: 0.7195 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1425 | Classification: 0.1044 | Classification_binary: 0.2020 | HitGrouping: 0.2380 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2017 | Regression_eta: 0.0023 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2116 | Regression_cos_phi: 0.1096 | Regression_energy: 0.0900 | Regression_eta: 0.0975 | Regression_pt: 0.0989 | Regression_sin_phi: 0.1529 | LR: 6.29e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.6553s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.6837 | max_loss: 0.4098 | max_backward: 1.4344 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.0218 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7022.8 | uss_MB: 6440.7 | child_rss_MB: 5233.8 | allocated_MB: 342.1 | reserved_MB: 24074.0 | inactive_split_MB: 269.3 +rank7 [2026-10-29 03:06:28,834] INFO : Step 22300/50000 rank7 | Train Loss: 1.4347 | Train Loss Unweighted: 0.7422 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1435 | Classification: 0.1142 | Classification_binary: 0.2209 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2003 | Regression_eta: 0.0024 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0979 | Regression_eta: 0.0779 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0731 | LR: 6.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8525s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6575 | max_loss: 0.2482 | max_backward: 1.2318 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7320 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7027.8 | uss_MB: 6444.8 | child_rss_MB: 5212.5 | allocated_MB: 344.0 | reserved_MB: 25712.0 | inactive_split_MB: 238.0 +rank7 [2026-10-29 03:10:23,118] INFO : Step 22400/50000 rank7 | Train Loss: 1.5901 | Train Loss Unweighted: 0.8194 | Train Loss Components: Auxiliary: 0.3279 | Cardinality: 0.1875 | Classification: 0.1234 | Classification_binary: 0.2689 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2092 | Regression_eta: 0.0025 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2743 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0870 | Regression_eta: 0.0847 | Regression_pt: 0.0914 | Regression_sin_phi: 0.1069 | LR: 6.23e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0084 | max_batch_to_device: 0.0068 | max_forward: 1.1355 | max_loss: 0.2280 | max_backward: 1.9589 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 7021.9 | uss_MB: 6441.2 | child_rss_MB: 5226.2 | allocated_MB: 344.4 | reserved_MB: 25712.0 | inactive_split_MB: 238.2 +rank7 [2026-10-29 03:14:16,069] INFO : Step 22500/50000 rank7 | Train Loss: 1.4744 | Train Loss Unweighted: 0.7666 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1672 | Classification: 0.1081 | Classification_binary: 0.2168 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2182 | Regression_eta: 0.0022 | Regression_pt: 0.2187 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1978 | Regression_cos_phi: 0.1131 | Regression_energy: 0.1219 | Regression_eta: 0.0689 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0660 | LR: 6.20e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8996s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0993 | max_forward: 0.7724 | max_loss: 0.1456 | max_backward: 1.3236 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1000 | Memory: rss_MB: 7027.9 | uss_MB: 6445.1 | child_rss_MB: 5225.9 | allocated_MB: 342.9 | reserved_MB: 25712.0 | inactive_split_MB: 211.8 +rank7 [2026-10-29 03:18:06,938] INFO : Step 22600/50000 rank7 | Train Loss: 1.3998 | Train Loss Unweighted: 0.7236 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1589 | Classification: 0.1046 | Classification_binary: 0.2029 | HitGrouping: 0.2349 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2043 | Regression_eta: 0.0023 | Regression_pt: 0.2073 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0734 | Regression_energy: 0.1044 | Regression_eta: 0.0641 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0742 | LR: 6.18e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5798s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0875 | max_forward: 0.9134 | max_loss: 0.2380 | max_backward: 1.9248 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.5123 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7027.9 | uss_MB: 5741.6 | child_rss_MB: 5221.9 | allocated_MB: 342.6 | reserved_MB: 25712.0 | inactive_split_MB: 160.7 +rank7 [2026-10-29 03:22:04,589] INFO : Step 22700/50000 rank7 | Train Loss: 1.5212 | Train Loss Unweighted: 0.7856 | Train Loss Components: Auxiliary: 0.3172 | Cardinality: 0.1717 | Classification: 0.1186 | Classification_binary: 0.2509 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2054 | Regression_eta: 0.0022 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2716 | Regression_cos_phi: 0.0890 | Regression_energy: 0.1096 | Regression_eta: 0.0770 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0797 | LR: 6.15e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3276s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8006 | max_loss: 0.1552 | max_backward: 1.6127 | max_optimizer_step: 0.3057 | max_tensorboard_logging: 0.6020 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 7027.7 | uss_MB: 6447.7 | child_rss_MB: 5222.8 | allocated_MB: 345.1 | reserved_MB: 26134.0 | inactive_split_MB: 242.2 +rank7 [2026-10-29 03:26:06,869] INFO : Step 22800/50000 rank7 | Train Loss: 1.5082 | Train Loss Unweighted: 0.7782 | Train Loss Components: Auxiliary: 0.3090 | Cardinality: 0.1634 | Classification: 0.1049 | Classification_binary: 0.2415 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2133 | Regression_eta: 0.0025 | Regression_pt: 0.2134 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0916 | Regression_energy: 0.1031 | Regression_eta: 0.0771 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0936 | LR: 6.12e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2792s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7725 | max_loss: 0.2465 | max_backward: 2.3058 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6047 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 7026.7 | uss_MB: 5960.8 | child_rss_MB: 5224.5 | allocated_MB: 345.5 | reserved_MB: 26726.0 | inactive_split_MB: 280.6 +rank7 [2026-10-29 03:30:17,846] INFO : Step 22900/50000 rank7 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6900 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1247 | Classification: 0.1019 | Classification_binary: 0.1897 | HitGrouping: 0.2263 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1958 | Regression_eta: 0.0020 | Regression_pt: 0.1985 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0891 | Regression_eta: 0.0838 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0762 | LR: 6.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6090s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6420 | max_loss: 0.2445 | max_backward: 1.1781 | max_optimizer_step: 0.3238 | max_tensorboard_logging: 2.4252 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0957 | Memory: rss_MB: 7027.8 | uss_MB: 6448.6 | child_rss_MB: 5203.8 | allocated_MB: 345.4 | reserved_MB: 27750.0 | inactive_split_MB: 267.3 +rank7 [2026-10-29 03:34:03,731] INFO : Step 23000/50000 rank7 | Train Loss: 1.7099 | Train Loss Unweighted: 0.8922 | Train Loss Components: Auxiliary: 0.3469 | Cardinality: 0.1952 | Classification: 0.1396 | Classification_binary: 0.2732 | HitGrouping: 0.2708 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2358 | Regression_eta: 0.0028 | Regression_pt: 0.2380 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2911 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1190 | Regression_eta: 0.1140 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1281 | LR: 6.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6405s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7806 | max_loss: 0.1477 | max_backward: 1.8333 | max_optimizer_step: 0.3249 | max_tensorboard_logging: 0.7706 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7028.8 | uss_MB: 6454.2 | child_rss_MB: 5205.6 | allocated_MB: 344.3 | reserved_MB: 27750.0 | inactive_split_MB: 227.8 +rank7 [2026-10-29 03:34:04,743] INFO : Running validation on rank7 +rank7 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 03:34:04,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 03:34:52,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:34:53,014] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 03:35:18,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 03:35:19,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 03:35:20,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 03:35:21,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 03:35:31,894] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 03:35:32,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:35:32,951] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 03:35:48,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 03:35:49,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 03:35:50,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 03:35:51,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 03:36:01,705] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 03:36:02,378] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:36:02,911] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 03:36:18,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 03:36:19,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 03:36:19,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 03:36:20,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_23000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 03:36:30,293] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 03:40:33,162] INFO : Step 23100/50000 rank7 | Train Loss: 1.5795 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3226 | Cardinality: 0.1952 | Classification: 0.1171 | Classification_binary: 0.2575 | HitGrouping: 0.2389 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2058 | Regression_eta: 0.0026 | Regression_pt: 0.2090 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2410 | Regression_cos_phi: 0.0973 | Regression_energy: 0.0945 | Regression_eta: 0.0683 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0775 | LR: 6.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4140s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8066 | max_loss: 0.1535 | max_backward: 1.5909 | max_optimizer_step: 0.3015 | max_tensorboard_logging: 0.6069 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0673 | Memory: rss_MB: 8901.5 | uss_MB: 8319.1 | child_rss_MB: 5229.7 | allocated_MB: 343.7 | reserved_MB: 23312.0 | inactive_split_MB: 248.6 +rank7 [2026-10-29 03:44:14,417] INFO : Step 23200/50000 rank7 | Train Loss: 1.3231 | Train Loss Unweighted: 0.6808 | Train Loss Components: Auxiliary: 0.2679 | Cardinality: 0.1301 | Classification: 0.1041 | Classification_binary: 0.1932 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1890 | Regression_eta: 0.0022 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0973 | Regression_energy: 0.0895 | Regression_eta: 0.0703 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0647 | LR: 6.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4827s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7215 | max_loss: 0.2348 | max_backward: 1.5168 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.9975 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7134.4 | uss_MB: 6551.7 | child_rss_MB: 5230.1 | allocated_MB: 340.9 | reserved_MB: 23784.0 | inactive_split_MB: 256.0 +rank7 [2026-10-29 03:47:55,308] INFO : Step 23300/50000 rank7 | Train Loss: 1.5792 | Train Loss Unweighted: 0.8223 | Train Loss Components: Auxiliary: 0.3171 | Cardinality: 0.1490 | Classification: 0.1210 | Classification_binary: 0.2386 | HitGrouping: 0.2598 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2279 | Regression_eta: 0.0031 | Regression_pt: 0.2290 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2150 | Regression_cos_phi: 0.0821 | Regression_energy: 0.1101 | Regression_eta: 0.0898 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0799 | LR: 5.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9637s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7459 | max_loss: 0.2470 | max_backward: 1.6098 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7139.4 | uss_MB: 6561.0 | child_rss_MB: 5217.6 | allocated_MB: 343.5 | reserved_MB: 25340.0 | inactive_split_MB: 254.5 +rank7 [2026-10-29 03:51:33,798] INFO : Step 23400/50000 rank7 | Train Loss: 1.4838 | Train Loss Unweighted: 0.7716 | Train Loss Components: Auxiliary: 0.3046 | Cardinality: 0.1536 | Classification: 0.1172 | Classification_binary: 0.2539 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1969 | Regression_eta: 0.0023 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2624 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0850 | Regression_eta: 0.0845 | Regression_pt: 0.0902 | Regression_sin_phi: 0.1084 | LR: 5.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1980s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0294 | max_forward: 0.8185 | max_loss: 0.1572 | max_backward: 1.5005 | max_optimizer_step: 0.2994 | max_tensorboard_logging: 0.5942 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7138.4 | uss_MB: 6561.0 | child_rss_MB: 5214.8 | allocated_MB: 349.6 | reserved_MB: 25806.0 | inactive_split_MB: 265.6 +rank7 [2026-10-29 03:55:22,654] INFO : Step 23500/50000 rank7 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7346 | Train Loss Components: Auxiliary: 0.2890 | Cardinality: 0.1357 | Classification: 0.0947 | Classification_binary: 0.2224 | HitGrouping: 0.2379 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2070 | Regression_eta: 0.0021 | Regression_pt: 0.2059 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2124 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0979 | Regression_eta: 0.0656 | Regression_pt: 0.1020 | Regression_sin_phi: 0.1041 | LR: 5.92e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0137s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7053 | max_loss: 0.1487 | max_backward: 1.3570 | max_optimizer_step: 0.3133 | max_tensorboard_logging: 0.6009 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7127.9 | uss_MB: 6561.7 | child_rss_MB: 5167.8 | allocated_MB: 345.3 | reserved_MB: 25806.0 | inactive_split_MB: 249.0 +rank7 [2026-10-29 03:59:15,167] INFO : Step 23600/50000 rank7 | Train Loss: 1.3412 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1488 | Classification: 0.1029 | Classification_binary: 0.1895 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2059 | Regression_eta: 0.0019 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0654 | Regression_energy: 0.1010 | Regression_eta: 0.0511 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0688 | LR: 5.90e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.9373s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7122 | max_loss: 0.1524 | max_backward: 1.4818 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 7100.9 | uss_MB: 6562.6 | child_rss_MB: 5053.8 | allocated_MB: 343.1 | reserved_MB: 25806.0 | inactive_split_MB: 162.3 +rank7 [2026-10-29 04:03:05,413] INFO : Step 23700/50000 rank7 | Train Loss: 1.4341 | Train Loss Unweighted: 0.7393 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1543 | Classification: 0.1113 | Classification_binary: 0.2366 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1918 | Regression_eta: 0.0022 | Regression_pt: 0.1950 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2252 | Regression_cos_phi: 0.1221 | Regression_energy: 0.0963 | Regression_eta: 0.0863 | Regression_pt: 0.1005 | Regression_sin_phi: 0.1017 | LR: 5.87e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3837s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1519 | max_backward: 1.8015 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 7069.7 | uss_MB: 6564.4 | child_rss_MB: 5057.6 | allocated_MB: 344.7 | reserved_MB: 25806.0 | inactive_split_MB: 258.1 +rank7 [2026-10-29 04:06:50,322] INFO : Step 23800/50000 rank7 | Train Loss: 1.5495 | Train Loss Unweighted: 0.7863 | Train Loss Components: Auxiliary: 0.3128 | Cardinality: 0.1774 | Classification: 0.1078 | Classification_binary: 0.2455 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2138 | Regression_eta: 0.0026 | Regression_pt: 0.2139 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2419 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0982 | Regression_eta: 0.0729 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0832 | LR: 5.84e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.0040s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7487 | max_loss: 0.2476 | max_backward: 1.5116 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 7059.7 | uss_MB: 6567.2 | child_rss_MB: 5057.5 | allocated_MB: 346.1 | reserved_MB: 25806.0 | inactive_split_MB: 203.1 +rank7 [2026-10-29 04:11:04,731] INFO : Step 23900/50000 rank7 | Train Loss: 1.3211 | Train Loss Unweighted: 0.7035 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1273 | Classification: 0.0972 | Classification_binary: 0.1923 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2033 | Regression_eta: 0.0018 | Regression_pt: 0.2068 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0938 | Regression_eta: 0.0588 | Regression_pt: 0.1008 | Regression_sin_phi: 0.1030 | LR: 5.81e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3368s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1089 | max_forward: 0.7511 | max_loss: 0.1421 | max_backward: 1.4326 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7054.1 | uss_MB: 6567.1 | child_rss_MB: 5030.1 | allocated_MB: 343.6 | reserved_MB: 26828.0 | inactive_split_MB: 273.7 +rank7 [2026-10-29 04:14:57,721] INFO : Step 24000/50000 rank7 | Train Loss: 1.4637 | Train Loss Unweighted: 0.7560 | Train Loss Components: Auxiliary: 0.3029 | Cardinality: 0.1597 | Classification: 0.1183 | Classification_binary: 0.2251 | HitGrouping: 0.2447 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2023 | Regression_eta: 0.0022 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0883 | Regression_eta: 0.0704 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0884 | LR: 5.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1439s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7693 | max_loss: 0.1567 | max_backward: 1.5105 | max_optimizer_step: 0.2950 | max_tensorboard_logging: 0.6950 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7055.1 | uss_MB: 6567.1 | child_rss_MB: 5024.7 | allocated_MB: 344.3 | reserved_MB: 26828.0 | inactive_split_MB: 261.7 +rank7 [2026-10-29 04:14:58,739] INFO : Running validation on rank7 +rank7 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:14:58,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:15:47,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:15:48,539] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 04:16:15,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:16:16,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:16:17,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:16:18,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:16:28,466] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:16:28,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:16:29,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 04:16:45,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:16:46,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:16:47,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:16:48,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:16:58,200] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:16:59,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:16:59,583] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 04:17:15,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:17:15,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:17:16,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:17:17,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_24000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:17:27,079] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 04:21:44,038] INFO : Step 24100/50000 rank7 | Train Loss: 1.5542 | Train Loss Unweighted: 0.8126 | Train Loss Components: Auxiliary: 0.3272 | Cardinality: 0.1680 | Classification: 0.1154 | Classification_binary: 0.2606 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2147 | Regression_eta: 0.0023 | Regression_pt: 0.2171 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0805 | Regression_energy: 0.1076 | Regression_eta: 0.0643 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0684 | LR: 5.75e-04 | DataLoad Time: 0.0205s | Model Forward Time: 7.7516s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9305 | max_loss: 0.1737 | max_backward: 1.8680 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 3.4987 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1391 | Memory: rss_MB: 8910.9 | uss_MB: 8422.0 | child_rss_MB: 5025.3 | allocated_MB: 346.4 | reserved_MB: 23654.0 | inactive_split_MB: 251.6 +rank7 [2026-10-29 04:25:33,906] INFO : Step 24200/50000 rank7 | Train Loss: 1.4009 | Train Loss Unweighted: 0.7388 | Train Loss Components: Auxiliary: 0.2929 | Cardinality: 0.1360 | Classification: 0.1079 | Classification_binary: 0.2078 | HitGrouping: 0.2434 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2088 | Regression_eta: 0.0020 | Regression_pt: 0.2099 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.1958 | Regression_cos_phi: 0.1235 | Regression_energy: 0.0958 | Regression_eta: 0.0792 | Regression_pt: 0.0994 | Regression_sin_phi: 0.1226 | LR: 5.73e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.2954s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0058 | max_forward: 0.7414 | max_loss: 0.1916 | max_backward: 1.4957 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.7396 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0714 | Memory: rss_MB: 8908.9 | uss_MB: 7713.7 | child_rss_MB: 5018.4 | allocated_MB: 339.7 | reserved_MB: 25612.0 | inactive_split_MB: 272.6 +rank7 [2026-10-29 04:29:22,270] INFO : Step 24300/50000 rank7 | Train Loss: 1.4947 | Train Loss Unweighted: 0.7866 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1609 | Classification: 0.1217 | Classification_binary: 0.2053 | HitGrouping: 0.2612 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2263 | Regression_eta: 0.0021 | Regression_pt: 0.2288 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0724 | Regression_energy: 0.1162 | Regression_eta: 0.0585 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0948 | LR: 5.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0405s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6444 | max_loss: 0.1536 | max_backward: 1.3144 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7119.2 | uss_MB: 6632.6 | child_rss_MB: 5017.9 | allocated_MB: 342.4 | reserved_MB: 25616.0 | inactive_split_MB: 222.8 +rank7 [2026-10-29 04:33:07,494] INFO : Step 24400/50000 rank7 | Train Loss: 1.9842 | Train Loss Unweighted: 0.9904 | Train Loss Components: Auxiliary: 0.4100 | Cardinality: 0.3162 | Classification: 0.1311 | Classification_binary: 0.2822 | HitGrouping: 0.2776 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2845 | Regression_eta: 0.0030 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2939 | Regression_cos_phi: 0.0893 | Regression_energy: 0.1597 | Regression_eta: 0.0915 | Regression_pt: 0.1641 | Regression_sin_phi: 0.1039 | LR: 5.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2373s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7467 | max_loss: 0.2252 | max_backward: 1.6315 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5041 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7118.8 | uss_MB: 6635.2 | child_rss_MB: 4998.9 | allocated_MB: 346.5 | reserved_MB: 26618.0 | inactive_split_MB: 251.6 +rank7 [2026-10-29 04:36:57,191] INFO : Step 24500/50000 rank7 | Train Loss: 1.6960 | Train Loss Unweighted: 0.9011 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1684 | Classification: 0.1214 | Classification_binary: 0.2647 | HitGrouping: 0.2851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2533 | Regression_eta: 0.0027 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2719 | Regression_cos_phi: 0.1123 | Regression_energy: 0.1111 | Regression_eta: 0.0812 | Regression_pt: 0.1171 | Regression_sin_phi: 0.1001 | LR: 5.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3260s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.7666 | max_loss: 0.1536 | max_backward: 1.5033 | max_optimizer_step: 0.2182 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7120.7 | uss_MB: 6634.0 | child_rss_MB: 5029.4 | allocated_MB: 345.9 | reserved_MB: 26618.0 | inactive_split_MB: 258.1 +rank7 [2026-10-29 04:40:40,561] INFO : Step 24600/50000 rank7 | Train Loss: 1.4539 | Train Loss Unweighted: 0.7791 | Train Loss Components: Auxiliary: 0.3067 | Cardinality: 0.1438 | Classification: 0.1070 | Classification_binary: 0.1995 | HitGrouping: 0.2606 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2331 | Regression_eta: 0.0022 | Regression_pt: 0.2350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0994 | Regression_eta: 0.0723 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0688 | LR: 5.61e-04 | DataLoad Time: 0.0265s | Model Forward Time: 3.9882s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6225 | max_loss: 0.2457 | max_backward: 1.2632 | max_optimizer_step: 0.2744 | max_tensorboard_logging: 0.7677 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 7122.7 | uss_MB: 6636.0 | child_rss_MB: 5023.1 | allocated_MB: 341.8 | reserved_MB: 27626.0 | inactive_split_MB: 240.5 +rank7 [2026-10-29 04:44:29,988] INFO : Step 24700/50000 rank7 | Train Loss: 1.8394 | Train Loss Unweighted: 0.9424 | Train Loss Components: Auxiliary: 0.3733 | Cardinality: 0.2051 | Classification: 0.1495 | Classification_binary: 0.2704 | HitGrouping: 0.2793 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2549 | Regression_eta: 0.0033 | Regression_pt: 0.2609 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2361 | Regression_cos_phi: 0.0972 | Regression_energy: 0.1170 | Regression_eta: 0.0935 | Regression_pt: 0.1225 | Regression_sin_phi: 0.1141 | LR: 5.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5082s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0073 | max_forward: 0.8518 | max_loss: 0.1633 | max_backward: 1.5301 | max_optimizer_step: 0.2142 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7122.0 | uss_MB: 6637.6 | child_rss_MB: 5016.4 | allocated_MB: 350.9 | reserved_MB: 27626.0 | inactive_split_MB: 236.5 +rank7 [2026-10-29 04:48:15,356] INFO : Step 24800/50000 rank7 | Train Loss: 1.5364 | Train Loss Unweighted: 0.8020 | Train Loss Components: Auxiliary: 0.3154 | Cardinality: 0.1643 | Classification: 0.1165 | Classification_binary: 0.2562 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2120 | Regression_eta: 0.0025 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2655 | Regression_cos_phi: 0.0966 | Regression_energy: 0.1014 | Regression_eta: 0.0875 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0956 | LR: 5.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8631s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8341 | max_loss: 0.1833 | max_backward: 1.8485 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.8481 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7123.0 | uss_MB: 6379.3 | child_rss_MB: 5021.3 | allocated_MB: 346.4 | reserved_MB: 27626.0 | inactive_split_MB: 247.5 +rank7 [2026-10-29 04:52:00,598] INFO : Step 24900/50000 rank7 | Train Loss: 1.3758 | Train Loss Unweighted: 0.7481 | Train Loss Components: Auxiliary: 0.2860 | Cardinality: 0.1201 | Classification: 0.1058 | Classification_binary: 0.1988 | HitGrouping: 0.2495 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2202 | Regression_eta: 0.0021 | Regression_pt: 0.2189 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2317 | Regression_cos_phi: 0.0756 | Regression_energy: 0.1100 | Regression_eta: 0.0689 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0847 | LR: 5.53e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9785s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0060 | max_forward: 0.6678 | max_loss: 0.2101 | max_backward: 1.2154 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.8281 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7122.1 | uss_MB: 6401.8 | child_rss_MB: 5016.4 | allocated_MB: 341.8 | reserved_MB: 28654.0 | inactive_split_MB: 260.6 +rank7 [2026-10-29 04:55:44,334] INFO : Step 25000/50000 rank7 | Train Loss: 1.6261 | Train Loss Unweighted: 0.8796 | Train Loss Components: Auxiliary: 0.3330 | Cardinality: 0.1880 | Classification: 0.1117 | Classification_binary: 0.2274 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2663 | Regression_eta: 0.0024 | Regression_pt: 0.2691 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2287 | Regression_cos_phi: 0.1024 | Regression_energy: 0.1493 | Regression_eta: 0.0710 | Regression_pt: 0.1558 | Regression_sin_phi: 0.1089 | LR: 5.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2492s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 0.6791 | max_loss: 0.2437 | max_backward: 1.6196 | max_optimizer_step: 0.2241 | max_tensorboard_logging: 0.7437 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 7124.1 | uss_MB: 6641.4 | child_rss_MB: 5021.9 | allocated_MB: 345.5 | reserved_MB: 28654.0 | inactive_split_MB: 239.8 +rank7 [2026-10-29 04:55:45,304] INFO : Running validation on rank7 +rank7 [2026-10-29 04:55:45,304] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:55:45,305] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:56:33,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:56:34,565] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 04:57:00,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:57:01,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:57:02,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:57:03,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:57:13,169] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:57:13,409] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:57:13,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 04:57:30,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:57:31,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:57:32,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:57:33,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:57:43,417] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:57:43,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:57:44,416] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 04:57:59,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:58:00,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:58:01,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:58:01,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_25000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:58:11,790] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 05:02:15,912] INFO : Step 25100/50000 rank7 | Train Loss: 1.5497 | Train Loss Unweighted: 0.7971 | Train Loss Components: Auxiliary: 0.3187 | Cardinality: 0.1687 | Classification: 0.1112 | Classification_binary: 0.2629 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2069 | Regression_eta: 0.0025 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.2766 | Regression_cos_phi: 0.0854 | Regression_energy: 0.1007 | Regression_eta: 0.0709 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0957 | LR: 5.47e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.7067s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7819 | max_loss: 0.1629 | max_backward: 1.7093 | max_optimizer_step: 0.2503 | max_tensorboard_logging: 0.8758 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0682 | Memory: rss_MB: 9044.1 | uss_MB: 8559.6 | child_rss_MB: 5024.2 | allocated_MB: 347.9 | reserved_MB: 23514.0 | inactive_split_MB: 259.9 +rank7 [2026-10-29 05:06:02,044] INFO : Step 25200/50000 rank7 | Train Loss: 1.5431 | Train Loss Unweighted: 0.7738 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1761 | Classification: 0.0981 | Classification_binary: 0.2286 | HitGrouping: 0.2716 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2208 | Regression_eta: 0.0024 | Regression_pt: 0.2212 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2729 | Regression_cos_phi: 0.1517 | Regression_energy: 0.1143 | Regression_eta: 0.1071 | Regression_pt: 0.1188 | Regression_sin_phi: 0.1346 | LR: 5.44e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2812s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.9018 | max_loss: 0.1459 | max_backward: 1.3040 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0678 | Memory: rss_MB: 9043.9 | uss_MB: 8561.7 | child_rss_MB: 5049.3 | allocated_MB: 340.0 | reserved_MB: 24618.0 | inactive_split_MB: 250.9 +rank7 [2026-10-29 05:09:52,834] INFO : Step 25300/50000 rank7 | Train Loss: 1.4517 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2978 | Cardinality: 0.1512 | Classification: 0.1059 | Classification_binary: 0.2056 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2242 | Regression_eta: 0.0022 | Regression_pt: 0.2267 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2453 | Regression_cos_phi: 0.0811 | Regression_energy: 0.1049 | Regression_eta: 0.0897 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1286 | LR: 5.42e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8554s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0496 | max_forward: 0.6569 | max_loss: 0.1419 | max_backward: 1.2344 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.6866 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0852 | Memory: rss_MB: 7225.2 | uss_MB: 6741.4 | child_rss_MB: 5045.8 | allocated_MB: 341.2 | reserved_MB: 24618.0 | inactive_split_MB: 234.8 +rank7 [2026-10-29 05:13:35,656] INFO : Step 25400/50000 rank7 | Train Loss: 1.8458 | Train Loss Unweighted: 0.9413 | Train Loss Components: Auxiliary: 0.3573 | Cardinality: 0.1942 | Classification: 0.1444 | Classification_binary: 0.2877 | HitGrouping: 0.2880 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2495 | Regression_eta: 0.0031 | Regression_pt: 0.2530 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.3059 | Regression_cos_phi: 0.1537 | Regression_energy: 0.1136 | Regression_eta: 0.1267 | Regression_pt: 0.1189 | Regression_sin_phi: 0.1815 | LR: 5.39e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0667s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.7981 | max_loss: 0.1503 | max_backward: 1.8212 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7207.1 | uss_MB: 6703.6 | child_rss_MB: 5012.2 | allocated_MB: 345.1 | reserved_MB: 24618.0 | inactive_split_MB: 264.9 +rank7 [2026-10-29 05:17:27,534] INFO : Step 25500/50000 rank7 | Train Loss: 1.5896 | Train Loss Unweighted: 0.8068 | Train Loss Components: Auxiliary: 0.3236 | Cardinality: 0.2213 | Classification: 0.1054 | Classification_binary: 0.2575 | HitGrouping: 0.2474 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2179 | Regression_eta: 0.0023 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2537 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0911 | Regression_eta: 0.0771 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0600 | LR: 5.36e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0756s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7435 | max_loss: 0.1471 | max_backward: 1.4139 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.6663 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7205.9 | uss_MB: 6744.1 | child_rss_MB: 5045.6 | allocated_MB: 341.8 | reserved_MB: 25622.0 | inactive_split_MB: 249.3 +rank7 [2026-10-29 05:21:16,671] INFO : Step 25600/50000 rank7 | Train Loss: 1.5518 | Train Loss Unweighted: 0.8427 | Train Loss Components: Auxiliary: 0.3219 | Cardinality: 0.1511 | Classification: 0.1127 | Classification_binary: 0.2133 | HitGrouping: 0.2713 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2556 | Regression_eta: 0.0026 | Regression_pt: 0.2561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0813 | Regression_energy: 0.1250 | Regression_eta: 0.0575 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0762 | LR: 5.33e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8752s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6361 | max_loss: 0.2365 | max_backward: 1.3806 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.6085 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7206.9 | uss_MB: 6746.4 | child_rss_MB: 5055.9 | allocated_MB: 339.8 | reserved_MB: 25622.0 | inactive_split_MB: 228.2 +rank7 [2026-10-29 05:25:00,204] INFO : Step 25700/50000 rank7 | Train Loss: 1.6733 | Train Loss Unweighted: 0.8854 | Train Loss Components: Auxiliary: 0.3513 | Cardinality: 0.2050 | Classification: 0.1262 | Classification_binary: 0.2574 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2469 | Regression_eta: 0.0022 | Regression_pt: 0.2499 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.2723 | Regression_cos_phi: 0.1233 | Regression_energy: 0.1191 | Regression_eta: 0.0959 | Regression_pt: 0.1250 | Regression_sin_phi: 0.1151 | LR: 5.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8869s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7603 | max_loss: 0.1440 | max_backward: 1.4469 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7204.7 | uss_MB: 6750.8 | child_rss_MB: 5054.2 | allocated_MB: 346.9 | reserved_MB: 26562.0 | inactive_split_MB: 217.1 +rank7 [2026-10-29 05:28:44,678] INFO : Step 25800/50000 rank7 | Train Loss: 1.8620 | Train Loss Unweighted: 0.9881 | Train Loss Components: Auxiliary: 0.3839 | Cardinality: 0.2074 | Classification: 0.1417 | Classification_binary: 0.3096 | HitGrouping: 0.2834 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2633 | Regression_eta: 0.0029 | Regression_pt: 0.2675 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.3018 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1170 | Regression_eta: 0.0984 | Regression_pt: 0.1219 | Regression_sin_phi: 0.1032 | LR: 5.27e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5182s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8292 | max_loss: 0.2560 | max_backward: 1.9693 | max_optimizer_step: 0.2209 | max_tensorboard_logging: 0.5004 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7204.7 | uss_MB: 6750.7 | child_rss_MB: 5051.9 | allocated_MB: 345.7 | reserved_MB: 26634.0 | inactive_split_MB: 284.3 +rank7 [2026-10-29 05:32:33,908] INFO : Step 25900/50000 rank7 | Train Loss: 1.5145 | Train Loss Unweighted: 0.7767 | Train Loss Components: Auxiliary: 0.3020 | Cardinality: 0.1555 | Classification: 0.0957 | Classification_binary: 0.2194 | HitGrouping: 0.2703 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2273 | Regression_eta: 0.0024 | Regression_pt: 0.2292 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2031 | Regression_cos_phi: 0.1037 | Regression_energy: 0.1196 | Regression_eta: 0.0919 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0994 | LR: 5.25e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8723s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7497 | max_loss: 0.1416 | max_backward: 1.2938 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.5680 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 7198.9 | uss_MB: 6742.8 | child_rss_MB: 5024.9 | allocated_MB: 343.3 | reserved_MB: 26634.0 | inactive_split_MB: 220.7 +rank7 [2026-10-29 05:36:28,636] INFO : Step 26000/50000 rank7 | Train Loss: 1.5398 | Train Loss Unweighted: 0.8285 | Train Loss Components: Auxiliary: 0.3142 | Cardinality: 0.1359 | Classification: 0.1217 | Classification_binary: 0.2149 | HitGrouping: 0.2580 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2437 | Regression_eta: 0.0028 | Regression_pt: 0.2427 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.2217 | Regression_cos_phi: 0.0920 | Regression_energy: 0.1222 | Regression_eta: 0.0768 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1082 | LR: 5.22e-04 | DataLoad Time: 0.0185s | Model Forward Time: 9.2325s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1135 | max_forward: 0.7289 | max_loss: 0.1414 | max_backward: 1.3108 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 5.7247 | max_tensorboard_flush: 0.0920 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 7205.9 | uss_MB: 6751.2 | child_rss_MB: 5050.8 | allocated_MB: 342.9 | reserved_MB: 26634.0 | inactive_split_MB: 247.9 +rank7 [2026-10-29 05:36:29,553] INFO : Running validation on rank7 +rank7 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 05:36:29,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 05:37:19,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:37:20,020] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 05:37:45,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 05:37:45,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 05:37:46,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 05:37:47,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 05:37:57,946] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 05:37:58,606] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:37:59,109] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 05:38:15,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 05:38:16,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 05:38:17,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 05:38:18,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 05:38:28,271] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 05:38:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:38:28,891] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 05:38:44,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 05:38:45,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 05:38:45,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 05:38:46,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_26000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 05:38:56,299] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 05:43:04,441] INFO : Step 26100/50000 rank7 | Train Loss: 1.7473 | Train Loss Unweighted: 0.8665 | Train Loss Components: Auxiliary: 0.3518 | Cardinality: 0.2605 | Classification: 0.1197 | Classification_binary: 0.2670 | HitGrouping: 0.2560 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2357 | Regression_eta: 0.0024 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2636 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1226 | Regression_eta: 0.0752 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0797 | LR: 5.19e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8575s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.8758 | max_loss: 0.1481 | max_backward: 1.7925 | max_optimizer_step: 0.3229 | max_tensorboard_logging: 0.7859 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0781 | Memory: rss_MB: 9058.7 | uss_MB: 8607.0 | child_rss_MB: 5018.3 | allocated_MB: 343.7 | reserved_MB: 22672.0 | inactive_split_MB: 246.3 +rank7 [2026-10-29 05:47:03,508] INFO : Step 26200/50000 rank7 | Train Loss: 1.4585 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3051 | Cardinality: 0.1739 | Classification: 0.0970 | Classification_binary: 0.2167 | HitGrouping: 0.2523 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2233 | Regression_eta: 0.0019 | Regression_pt: 0.2251 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2318 | Regression_cos_phi: 0.0774 | Regression_energy: 0.1081 | Regression_eta: 0.0585 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0561 | LR: 5.16e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.2729s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0063 | max_forward: 0.7349 | max_loss: 0.2422 | max_backward: 1.5770 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 9059.7 | uss_MB: 8609.1 | child_rss_MB: 5012.6 | allocated_MB: 342.6 | reserved_MB: 23692.0 | inactive_split_MB: 252.5 +rank7 [2026-10-29 05:50:52,485] INFO : Step 26300/50000 rank7 | Train Loss: 1.4688 | Train Loss Unweighted: 0.7784 | Train Loss Components: Auxiliary: 0.3056 | Cardinality: 0.1560 | Classification: 0.1104 | Classification_binary: 0.2053 | HitGrouping: 0.2494 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2278 | Regression_eta: 0.0022 | Regression_pt: 0.2302 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0997 | Regression_eta: 0.0517 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0835 | LR: 5.13e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3841s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.6227 | max_loss: 0.2423 | max_backward: 1.3741 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7275.8 | uss_MB: 6548.4 | child_rss_MB: 5006.3 | allocated_MB: 343.7 | reserved_MB: 23692.0 | inactive_split_MB: 218.3 +rank7 [2026-10-29 05:54:52,696] INFO : Step 26400/50000 rank7 | Train Loss: 1.7770 | Train Loss Unweighted: 1.0065 | Train Loss Components: Auxiliary: 0.3726 | Cardinality: 0.1446 | Classification: 0.1401 | Classification_binary: 0.2730 | HitGrouping: 0.2674 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2922 | Regression_eta: 0.0030 | Regression_pt: 0.2950 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2903 | Regression_cos_phi: 0.1171 | Regression_energy: 0.1522 | Regression_eta: 0.1179 | Regression_pt: 0.1578 | Regression_sin_phi: 0.1611 | LR: 5.10e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3820s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0371 | max_forward: 0.7799 | max_loss: 0.1542 | max_backward: 1.5401 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8208 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 7278.3 | uss_MB: 6829.8 | child_rss_MB: 5007.4 | allocated_MB: 345.0 | reserved_MB: 24132.0 | inactive_split_MB: 292.0 +rank7 [2026-10-29 05:58:42,121] INFO : Step 26500/50000 rank7 | Train Loss: 1.6463 | Train Loss Unweighted: 0.8523 | Train Loss Components: Auxiliary: 0.3475 | Cardinality: 0.2250 | Classification: 0.0957 | Classification_binary: 0.2597 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2452 | Regression_eta: 0.0022 | Regression_pt: 0.2471 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.2406 | Regression_cos_phi: 0.0803 | Regression_energy: 0.1226 | Regression_eta: 0.0697 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0767 | LR: 5.08e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.4048s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.7738 | max_loss: 0.1498 | max_backward: 1.4041 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.9167 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7270.0 | uss_MB: 6831.7 | child_rss_MB: 4958.2 | allocated_MB: 344.7 | reserved_MB: 24132.0 | inactive_split_MB: 253.3 +rank7 [2026-10-29 06:02:41,778] INFO : Step 26600/50000 rank7 | Train Loss: 1.4656 | Train Loss Unweighted: 0.7951 | Train Loss Components: Auxiliary: 0.3064 | Cardinality: 0.1429 | Classification: 0.1073 | Classification_binary: 0.1948 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2437 | Regression_eta: 0.0021 | Regression_pt: 0.2449 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0751 | Regression_energy: 0.1150 | Regression_eta: 0.0769 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0860 | LR: 5.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7652s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0062 | max_forward: 1.0175 | max_loss: 0.2697 | max_backward: 1.7041 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.8432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1569 | Memory: rss_MB: 7268.2 | uss_MB: 6828.7 | child_rss_MB: 4963.8 | allocated_MB: 343.3 | reserved_MB: 24956.0 | inactive_split_MB: 251.5 +rank7 [2026-10-29 06:06:39,601] INFO : Step 26700/50000 rank7 | Train Loss: 1.6959 | Train Loss Unweighted: 0.8873 | Train Loss Components: Auxiliary: 0.3504 | Cardinality: 0.1928 | Classification: 0.1365 | Classification_binary: 0.2417 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2501 | Regression_eta: 0.0026 | Regression_pt: 0.2535 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2994 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1176 | Regression_eta: 0.0844 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0808 | LR: 5.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.6713s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 1.5783 | max_loss: 0.3860 | max_backward: 3.4594 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.9867 | max_tensorboard_flush: 0.0137 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7266.7 | uss_MB: 6826.0 | child_rss_MB: 4965.5 | allocated_MB: 345.1 | reserved_MB: 24956.0 | inactive_split_MB: 208.8 +rank7 [2026-10-29 06:10:41,253] INFO : Step 26800/50000 rank7 | Train Loss: 1.8364 | Train Loss Unweighted: 0.9836 | Train Loss Components: Auxiliary: 0.3837 | Cardinality: 0.2130 | Classification: 0.1318 | Classification_binary: 0.3002 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2717 | Regression_eta: 0.0028 | Regression_pt: 0.2741 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2972 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1477 | Regression_eta: 0.0835 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0958 | LR: 4.99e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.2414s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.7757 | max_loss: 0.1823 | max_backward: 1.6003 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1422 | Memory: rss_MB: 7265.7 | uss_MB: 6830.4 | child_rss_MB: 4931.6 | allocated_MB: 345.6 | reserved_MB: 24956.0 | inactive_split_MB: 269.3 +rank7 [2026-10-29 06:14:37,246] INFO : Step 26900/50000 rank7 | Train Loss: 1.4981 | Train Loss Unweighted: 0.8090 | Train Loss Components: Auxiliary: 0.3177 | Cardinality: 0.1667 | Classification: 0.0997 | Classification_binary: 0.2199 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2414 | Regression_eta: 0.0021 | Regression_pt: 0.2437 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2227 | Regression_cos_phi: 0.1077 | Regression_energy: 0.1193 | Regression_eta: 0.0870 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0971 | LR: 4.96e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.8903s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0248 | max_forward: 0.7236 | max_loss: 0.1466 | max_backward: 1.3543 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 0.5156 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0473 | Memory: rss_MB: 7267.0 | uss_MB: 6832.2 | child_rss_MB: 4930.7 | allocated_MB: 345.0 | reserved_MB: 25976.0 | inactive_split_MB: 262.8 +rank7 [2026-10-29 06:18:43,003] INFO : Step 27000/50000 rank7 | Train Loss: 1.4773 | Train Loss Unweighted: 0.7815 | Train Loss Components: Auxiliary: 0.3040 | Cardinality: 0.1427 | Classification: 0.1232 | Classification_binary: 0.2126 | HitGrouping: 0.2620 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2198 | Regression_eta: 0.0021 | Regression_pt: 0.2213 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2486 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0962 | Regression_eta: 0.0777 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0830 | LR: 4.94e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9877s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.7301 | max_loss: 0.1424 | max_backward: 1.7880 | max_optimizer_step: 0.2384 | max_tensorboard_logging: 0.6643 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7264.5 | uss_MB: 6140.3 | child_rss_MB: 4897.0 | allocated_MB: 344.6 | reserved_MB: 25976.0 | inactive_split_MB: 208.4 +rank7 [2026-10-29 06:18:43,933] INFO : Running validation on rank7 +rank7 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 06:18:43,933] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 06:19:32,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:19:33,148] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 06:20:00,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 06:20:01,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 06:20:02,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 06:20:03,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 06:20:13,507] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 06:20:14,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:20:14,781] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 06:20:31,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 06:20:32,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 06:20:34,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 06:20:35,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 06:20:45,013] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 06:20:45,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:20:46,292] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 06:21:01,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 06:21:02,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 06:21:03,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 06:21:03,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_27000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 06:21:13,674] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 06:25:28,749] INFO : Step 27100/50000 rank7 | Train Loss: 1.6210 | Train Loss Unweighted: 0.8787 | Train Loss Components: Auxiliary: 0.3365 | Cardinality: 0.1531 | Classification: 0.1235 | Classification_binary: 0.2630 | HitGrouping: 0.2552 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2416 | Regression_eta: 0.0026 | Regression_pt: 0.2452 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2492 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0951 | Regression_eta: 0.1074 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1196 | LR: 4.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7016s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0065 | max_forward: 0.8169 | max_loss: 0.1526 | max_backward: 1.6547 | max_optimizer_step: 0.2557 | max_tensorboard_logging: 0.8019 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1777 | Memory: rss_MB: 9120.7 | uss_MB: 8683.2 | child_rss_MB: 4899.0 | allocated_MB: 344.4 | reserved_MB: 21936.0 | inactive_split_MB: 255.7 +rank7 [2026-10-29 06:29:29,295] INFO : Step 27200/50000 rank7 | Train Loss: 1.5273 | Train Loss Unweighted: 0.8306 | Train Loss Components: Auxiliary: 0.3222 | Cardinality: 0.1509 | Classification: 0.1095 | Classification_binary: 0.2323 | HitGrouping: 0.2492 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2420 | Regression_eta: 0.0023 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1127 | Regression_eta: 0.0684 | Regression_pt: 0.1149 | Regression_sin_phi: 0.1092 | LR: 4.88e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.5735s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0174 | max_forward: 1.0046 | max_loss: 0.1755 | max_backward: 1.5948 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.7525 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 9122.9 | uss_MB: 8683.2 | child_rss_MB: 4899.3 | allocated_MB: 344.0 | reserved_MB: 22848.0 | inactive_split_MB: 274.0 +rank7 [2026-10-29 06:33:36,121] INFO : Step 27300/50000 rank7 | Train Loss: 1.3604 | Train Loss Unweighted: 0.7215 | Train Loss Components: Auxiliary: 0.2815 | Cardinality: 0.1429 | Classification: 0.0996 | Classification_binary: 0.1827 | HitGrouping: 0.2451 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2160 | Regression_eta: 0.0020 | Regression_pt: 0.2190 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0993 | Regression_eta: 0.0825 | Regression_pt: 0.1029 | Regression_sin_phi: 0.0911 | LR: 4.85e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.9945s | Phase Times: max_pre_step_barrier: 0.0050 | max_batch_to_device: 0.0068 | max_forward: 0.7457 | max_loss: 0.2323 | max_backward: 1.1512 | max_optimizer_step: 0.2913 | max_tensorboard_logging: 0.7781 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7350.6 | uss_MB: 6909.9 | child_rss_MB: 4892.8 | allocated_MB: 343.2 | reserved_MB: 23452.0 | inactive_split_MB: 260.1 +rank7 [2026-10-29 06:37:36,742] INFO : Step 27400/50000 rank7 | Train Loss: 1.6369 | Train Loss Unweighted: 0.8613 | Train Loss Components: Auxiliary: 0.3400 | Cardinality: 0.1743 | Classification: 0.1288 | Classification_binary: 0.2546 | HitGrouping: 0.2592 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2348 | Regression_eta: 0.0024 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2808 | Regression_cos_phi: 0.1416 | Regression_energy: 0.1285 | Regression_eta: 0.1279 | Regression_pt: 0.1345 | Regression_sin_phi: 0.1575 | LR: 4.82e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5095s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.7627 | max_loss: 0.1531 | max_backward: 1.5716 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.9391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0628 | Memory: rss_MB: 7346.4 | uss_MB: 6915.5 | child_rss_MB: 4881.9 | allocated_MB: 345.1 | reserved_MB: 23452.0 | inactive_split_MB: 224.2 +rank7 [2026-10-29 06:41:28,998] INFO : Step 27500/50000 rank7 | Train Loss: 1.6651 | Train Loss Unweighted: 0.8731 | Train Loss Components: Auxiliary: 0.3502 | Cardinality: 0.1900 | Classification: 0.1161 | Classification_binary: 0.2710 | HitGrouping: 0.2534 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2394 | Regression_eta: 0.0024 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2790 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1046 | Regression_eta: 0.0816 | Regression_pt: 0.1082 | Regression_sin_phi: 0.1453 | LR: 4.80e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9217s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0129 | max_forward: 1.5801 | max_loss: 0.2567 | max_backward: 2.0754 | max_optimizer_step: 0.2313 | max_tensorboard_logging: 0.6403 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7327.8 | uss_MB: 6909.6 | child_rss_MB: 4882.1 | allocated_MB: 345.3 | reserved_MB: 23452.0 | inactive_split_MB: 188.3 +rank7 [2026-10-29 06:45:23,370] INFO : Step 27600/50000 rank7 | Train Loss: 1.4036 | Train Loss Unweighted: 0.7482 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1301 | Classification: 0.1055 | Classification_binary: 0.1982 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2193 | Regression_eta: 0.0021 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1992 | Regression_cos_phi: 0.1203 | Regression_energy: 0.1071 | Regression_eta: 0.0675 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0837 | LR: 4.77e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8532s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0449 | max_forward: 0.7251 | max_loss: 0.1461 | max_backward: 1.2895 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0935 | Memory: rss_MB: 7330.5 | uss_MB: 6913.8 | child_rss_MB: 4883.3 | allocated_MB: 346.5 | reserved_MB: 23452.0 | inactive_split_MB: 265.7 +rank7 [2026-10-29 06:49:16,010] INFO : Step 27700/50000 rank7 | Train Loss: 1.5009 | Train Loss Unweighted: 0.8445 | Train Loss Components: Auxiliary: 0.3253 | Cardinality: 0.1384 | Classification: 0.1078 | Classification_binary: 0.2067 | HitGrouping: 0.2513 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2616 | Regression_eta: 0.0020 | Regression_pt: 0.2640 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2213 | Regression_cos_phi: 0.1215 | Regression_energy: 0.1141 | Regression_eta: 0.0797 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1256 | LR: 4.74e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0213s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.6835 | max_loss: 0.1439 | max_backward: 1.4030 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.1141 | Memory: rss_MB: 7272.5 | uss_MB: 6910.0 | child_rss_MB: 4816.1 | allocated_MB: 345.0 | reserved_MB: 23452.0 | inactive_split_MB: 248.8 +rank7 [2026-10-29 06:53:16,135] INFO : Step 27800/50000 rank7 | Train Loss: 1.6183 | Train Loss Unweighted: 0.8486 | Train Loss Components: Auxiliary: 0.3478 | Cardinality: 0.1714 | Classification: 0.1136 | Classification_binary: 0.2603 | HitGrouping: 0.2561 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2344 | Regression_eta: 0.0025 | Regression_pt: 0.2352 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.2665 | Regression_cos_phi: 0.1169 | Regression_energy: 0.1196 | Regression_eta: 0.0823 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0947 | LR: 4.71e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.6679s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0122 | max_forward: 1.2720 | max_loss: 0.2500 | max_backward: 2.1893 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.9238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7255.0 | uss_MB: 6628.1 | child_rss_MB: 4758.8 | allocated_MB: 346.2 | reserved_MB: 23452.0 | inactive_split_MB: 282.0 +rank7 [2026-10-29 06:57:17,960] INFO : Step 27900/50000 rank7 | Train Loss: 1.5061 | Train Loss Unweighted: 0.7994 | Train Loss Components: Auxiliary: 0.3136 | Cardinality: 0.1669 | Classification: 0.1081 | Classification_binary: 0.2209 | HitGrouping: 0.2586 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2326 | Regression_eta: 0.0023 | Regression_pt: 0.2332 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2375 | Regression_cos_phi: 0.0909 | Regression_energy: 0.1006 | Regression_eta: 0.0815 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0684 | LR: 4.68e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5369s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6739 | max_loss: 0.2496 | max_backward: 1.5159 | max_optimizer_step: 0.2307 | max_tensorboard_logging: 1.0584 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 7236.8 | uss_MB: 6915.2 | child_rss_MB: 4713.0 | allocated_MB: 342.9 | reserved_MB: 23452.0 | inactive_split_MB: 270.6 +rank7 [2026-10-29 07:01:10,308] INFO : Step 28000/50000 rank7 | Train Loss: 1.4297 | Train Loss Unweighted: 0.7873 | Train Loss Components: Auxiliary: 0.3026 | Cardinality: 0.1338 | Classification: 0.1103 | Classification_binary: 0.1973 | HitGrouping: 0.2454 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2375 | Regression_eta: 0.0022 | Regression_pt: 0.2377 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0667 | Regression_energy: 0.1088 | Regression_eta: 0.0517 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0873 | LR: 4.66e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.3448s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0063 | max_forward: 0.6371 | max_loss: 0.2435 | max_backward: 1.3543 | max_optimizer_step: 0.2274 | max_tensorboard_logging: 0.7744 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7234.1 | uss_MB: 6701.0 | child_rss_MB: 4698.6 | allocated_MB: 343.4 | reserved_MB: 23452.0 | inactive_split_MB: 256.6 +rank7 [2026-10-29 07:01:11,150] INFO : Running validation on rank7 +rank7 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:01:11,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:02:01,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:02:02,348] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 07:02:30,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:02:30,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:02:31,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:02:32,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:02:42,903] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:02:43,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:02:44,048] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 07:02:59,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:03:00,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:03:02,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:03:03,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:03:13,418] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 07:03:13,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:03:14,475] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 07:03:29,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:03:30,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:03:31,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:03:31,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_28000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:03:41,639] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 07:08:04,579] INFO : Step 28100/50000 rank7 | Train Loss: 1.6239 | Train Loss Unweighted: 0.8955 | Train Loss Components: Auxiliary: 0.3459 | Cardinality: 0.1553 | Classification: 0.1211 | Classification_binary: 0.2433 | HitGrouping: 0.2643 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2626 | Regression_eta: 0.0023 | Regression_pt: 0.2636 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2466 | Regression_cos_phi: 0.0901 | Regression_energy: 0.1383 | Regression_eta: 0.0750 | Regression_pt: 0.1433 | Regression_sin_phi: 0.0986 | LR: 4.63e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1490s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7436 | max_loss: 0.1474 | max_backward: 1.8400 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 1.0246 | max_tensorboard_flush: 0.0075 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 8987.7 | uss_MB: 7957.5 | child_rss_MB: 4681.2 | allocated_MB: 343.0 | reserved_MB: 21966.0 | inactive_split_MB: 177.6 +rank7 [2026-10-29 07:11:56,590] INFO : Step 28200/50000 rank7 | Train Loss: 1.5370 | Train Loss Unweighted: 0.8433 | Train Loss Components: Auxiliary: 0.3333 | Cardinality: 0.1488 | Classification: 0.1126 | Classification_binary: 0.2443 | HitGrouping: 0.2524 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2403 | Regression_eta: 0.0020 | Regression_pt: 0.2417 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2394 | Regression_cos_phi: 0.1060 | Regression_energy: 0.1116 | Regression_eta: 0.0641 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0978 | LR: 4.60e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2155s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1490 | max_backward: 1.5331 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0746 | Memory: rss_MB: 8988.3 | uss_MB: 8665.9 | child_rss_MB: 4706.2 | allocated_MB: 343.0 | reserved_MB: 23906.0 | inactive_split_MB: 271.1 +rank7 [2026-10-29 07:16:10,738] INFO : Step 28300/50000 rank7 | Train Loss: 1.4040 | Train Loss Unweighted: 0.7670 | Train Loss Components: Auxiliary: 0.2958 | Cardinality: 0.1377 | Classification: 0.1044 | Classification_binary: 0.1878 | HitGrouping: 0.2480 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2342 | Regression_eta: 0.0020 | Regression_pt: 0.2364 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1632 | Regression_cos_phi: 0.1083 | Regression_energy: 0.0949 | Regression_eta: 0.0576 | Regression_pt: 0.0979 | Regression_sin_phi: 0.1079 | LR: 4.57e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.8320s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6303 | max_loss: 0.2388 | max_backward: 1.3328 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7342.1 | uss_MB: 6460.8 | child_rss_MB: 4674.6 | allocated_MB: 342.9 | reserved_MB: 24950.0 | inactive_split_MB: 213.4 +rank7 [2026-10-29 07:20:03,894] INFO : Step 28400/50000 rank7 | Train Loss: 1.5695 | Train Loss Unweighted: 0.8253 | Train Loss Components: Auxiliary: 0.3321 | Cardinality: 0.1729 | Classification: 0.1116 | Classification_binary: 0.2228 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2414 | Regression_eta: 0.0022 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.2294 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1052 | Regression_eta: 0.0664 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1280 | LR: 4.55e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0890s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8374 | max_loss: 0.1569 | max_backward: 1.4906 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.5220 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1232 | Memory: rss_MB: 7340.1 | uss_MB: 7026.3 | child_rss_MB: 4704.3 | allocated_MB: 344.7 | reserved_MB: 24950.0 | inactive_split_MB: 223.1 +rank7 [2026-10-29 07:24:00,202] INFO : Step 28500/50000 rank7 | Train Loss: 1.6843 | Train Loss Unweighted: 0.9050 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.1816 | Classification: 0.1193 | Classification_binary: 0.2694 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2563 | Regression_eta: 0.0025 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2984 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1195 | Regression_eta: 0.0757 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0861 | LR: 4.52e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.1348s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1122 | max_forward: 0.8986 | max_loss: 0.1539 | max_backward: 1.8176 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 1.2305 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7336.4 | uss_MB: 7022.4 | child_rss_MB: 4702.6 | allocated_MB: 343.2 | reserved_MB: 24950.0 | inactive_split_MB: 270.6 +rank7 [2026-10-29 07:27:52,534] INFO : Step 28600/50000 rank7 | Train Loss: 1.4447 | Train Loss Unweighted: 0.7662 | Train Loss Components: Auxiliary: 0.3007 | Cardinality: 0.1669 | Classification: 0.1043 | Classification_binary: 0.2041 | HitGrouping: 0.2448 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2256 | Regression_eta: 0.0020 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2206 | Regression_cos_phi: 0.1120 | Regression_energy: 0.1120 | Regression_eta: 0.0867 | Regression_pt: 0.1170 | Regression_sin_phi: 0.1112 | LR: 4.49e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2898s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6768 | max_loss: 0.1698 | max_backward: 1.2681 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 1.0575 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 7341.0 | uss_MB: 7025.8 | child_rss_MB: 4687.0 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 265.8 +rank7 [2026-10-29 07:31:49,173] INFO : Step 28700/50000 rank7 | Train Loss: 1.4495 | Train Loss Unweighted: 0.7575 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1522 | Classification: 0.1065 | Classification_binary: 0.2102 | HitGrouping: 0.2525 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2169 | Regression_eta: 0.0020 | Regression_pt: 0.2195 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2187 | Regression_cos_phi: 0.1034 | Regression_energy: 0.1104 | Regression_eta: 0.0715 | Regression_pt: 0.1188 | Regression_sin_phi: 0.1084 | LR: 4.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0947s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8732 | max_loss: 0.2574 | max_backward: 2.1326 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.9053 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 7341.0 | uss_MB: 7028.2 | child_rss_MB: 4688.0 | allocated_MB: 343.9 | reserved_MB: 24950.0 | inactive_split_MB: 235.0 +rank7 [2026-10-29 07:35:44,674] INFO : Step 28800/50000 rank7 | Train Loss: 1.5168 | Train Loss Unweighted: 0.8022 | Train Loss Components: Auxiliary: 0.3277 | Cardinality: 0.1616 | Classification: 0.1102 | Classification_binary: 0.2325 | HitGrouping: 0.2412 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2269 | Regression_eta: 0.0021 | Regression_pt: 0.2279 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2405 | Regression_cos_phi: 0.1059 | Regression_energy: 0.0988 | Regression_eta: 0.0691 | Regression_pt: 0.1029 | Regression_sin_phi: 0.1029 | LR: 4.44e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4534s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.8379 | max_loss: 0.1575 | max_backward: 1.6001 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.7759 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 7342.0 | uss_MB: 7028.4 | child_rss_MB: 4682.4 | allocated_MB: 345.0 | reserved_MB: 24950.0 | inactive_split_MB: 270.4 +rank7 [2026-10-29 07:39:38,710] INFO : Step 28900/50000 rank7 | Train Loss: 1.4076 | Train Loss Unweighted: 0.7260 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1606 | Classification: 0.0945 | Classification_binary: 0.2125 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2068 | Regression_eta: 0.0019 | Regression_pt: 0.2081 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0597 | Regression_energy: 0.1002 | Regression_eta: 0.0511 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0903 | LR: 4.41e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.3189s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0287 | max_forward: 0.7473 | max_loss: 0.1493 | max_backward: 1.3700 | max_optimizer_step: 0.2297 | max_tensorboard_logging: 0.5269 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7346.0 | uss_MB: 7028.0 | child_rss_MB: 4692.4 | allocated_MB: 345.8 | reserved_MB: 24950.0 | inactive_split_MB: 290.1 +rank7 [2026-10-29 07:43:43,865] INFO : Step 29000/50000 rank7 | Train Loss: 1.3528 | Train Loss Unweighted: 0.7183 | Train Loss Components: Auxiliary: 0.2926 | Cardinality: 0.1404 | Classification: 0.0991 | Classification_binary: 0.1890 | HitGrouping: 0.2427 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2134 | Regression_eta: 0.0019 | Regression_pt: 0.2129 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2203 | Regression_cos_phi: 0.0960 | Regression_energy: 0.0837 | Regression_eta: 0.0832 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0924 | LR: 4.38e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9068s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1059 | max_forward: 0.7365 | max_loss: 0.1381 | max_backward: 1.3110 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.5212 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 7346.6 | uss_MB: 7029.7 | child_rss_MB: 4674.6 | allocated_MB: 349.5 | reserved_MB: 24950.0 | inactive_split_MB: 237.0 +rank7 [2026-10-29 07:43:44,819] INFO : Running validation on rank7 +rank7 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:43:44,819] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:44:33,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:44:33,713] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 07:44:59,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:45:00,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:45:01,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:45:02,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:45:12,489] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:45:12,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:45:13,371] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 07:45:29,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:45:30,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:45:31,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:45:32,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:45:42,222] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:45:43,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:45:43,719] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 07:45:58,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:45:59,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:45:59,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:46:00,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_29000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:46:10,553] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 07:50:42,138] INFO : Step 29100/50000 rank7 | Train Loss: 1.5189 | Train Loss Unweighted: 0.8004 | Train Loss Components: Auxiliary: 0.3278 | Cardinality: 0.1826 | Classification: 0.1126 | Classification_binary: 0.2345 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2222 | Regression_eta: 0.0019 | Regression_pt: 0.2268 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0958 | Regression_eta: 0.0586 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0824 | LR: 4.35e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.6997s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.8841 | max_loss: 0.1500 | max_backward: 1.4494 | max_optimizer_step: 0.2592 | max_tensorboard_logging: 1.0735 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0735 | Memory: rss_MB: 9019.5 | uss_MB: 8702.8 | child_rss_MB: 4660.9 | allocated_MB: 343.9 | reserved_MB: 22566.0 | inactive_split_MB: 261.8 +rank7 [2026-10-29 07:54:30,394] INFO : Step 29200/50000 rank7 | Train Loss: 1.4393 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1463 | Classification: 0.1049 | Classification_binary: 0.2251 | HitGrouping: 0.2404 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1992 | Regression_eta: 0.0023 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2231 | Regression_cos_phi: 0.0718 | Regression_energy: 0.1062 | Regression_eta: 0.0699 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0863 | LR: 4.33e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2676s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0182 | max_forward: 1.3981 | max_loss: 0.2525 | max_backward: 1.8956 | max_optimizer_step: 0.2608 | max_tensorboard_logging: 0.8985 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0774 | Memory: rss_MB: 9024.3 | uss_MB: 8708.4 | child_rss_MB: 4684.5 | allocated_MB: 346.4 | reserved_MB: 25564.0 | inactive_split_MB: 265.6 +rank7 [2026-10-29 07:58:22,885] INFO : Step 29300/50000 rank7 | Train Loss: 1.2553 | Train Loss Unweighted: 0.6725 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1195 | Classification: 0.0806 | Classification_binary: 0.1839 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2021 | Regression_eta: 0.0015 | Regression_pt: 0.2025 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0913 | Regression_eta: 0.0599 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0854 | LR: 4.30e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9982s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0618 | max_forward: 0.9607 | max_loss: 0.1849 | max_backward: 1.5306 | max_optimizer_step: 0.2545 | max_tensorboard_logging: 1.2772 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7449.5 | uss_MB: 7131.7 | child_rss_MB: 4690.4 | allocated_MB: 341.4 | reserved_MB: 25564.0 | inactive_split_MB: 228.6 +rank7 [2026-10-29 08:02:11,406] INFO : Step 29400/50000 rank7 | Train Loss: 1.4025 | Train Loss Unweighted: 0.7531 | Train Loss Components: Auxiliary: 0.3010 | Cardinality: 0.1371 | Classification: 0.1130 | Classification_binary: 0.2094 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2122 | Regression_eta: 0.0020 | Regression_pt: 0.2144 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0989 | Regression_eta: 0.0484 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0881 | LR: 4.27e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5174s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0085 | max_forward: 0.7514 | max_loss: 0.1408 | max_backward: 1.3690 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 1.1069 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7457.8 | uss_MB: 7141.4 | child_rss_MB: 4697.0 | allocated_MB: 345.6 | reserved_MB: 25564.0 | inactive_split_MB: 221.9 +rank7 [2026-10-29 08:05:54,734] INFO : Step 29500/50000 rank7 | Train Loss: 1.5076 | Train Loss Unweighted: 0.7920 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1472 | Classification: 0.1145 | Classification_binary: 0.2477 | HitGrouping: 0.2388 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2110 | Regression_eta: 0.0023 | Regression_pt: 0.2140 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2540 | Regression_cos_phi: 0.1435 | Regression_energy: 0.0959 | Regression_eta: 0.0986 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0792 | LR: 4.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4986s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7759 | max_loss: 0.1503 | max_backward: 1.4438 | max_optimizer_step: 0.2395 | max_tensorboard_logging: 0.9094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1358 | Memory: rss_MB: 7458.3 | uss_MB: 7143.2 | child_rss_MB: 4672.4 | allocated_MB: 342.6 | reserved_MB: 25564.0 | inactive_split_MB: 273.6 +rank7 [2026-10-29 08:09:49,625] INFO : Step 29600/50000 rank7 | Train Loss: 1.3018 | Train Loss Unweighted: 0.6836 | Train Loss Components: Auxiliary: 0.2799 | Cardinality: 0.1346 | Classification: 0.0846 | Classification_binary: 0.1909 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2021 | Regression_eta: 0.0018 | Regression_pt: 0.2023 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0983 | Regression_eta: 0.0420 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0523 | LR: 4.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7097s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6911 | max_loss: 0.2365 | max_backward: 1.6189 | max_optimizer_step: 0.2236 | max_tensorboard_logging: 1.1486 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 7455.4 | uss_MB: 7139.8 | child_rss_MB: 4697.9 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 213.4 +rank7 [2026-10-29 08:13:31,665] INFO : Step 29700/50000 rank7 | Train Loss: 1.2998 | Train Loss Unweighted: 0.6684 | Train Loss Components: Auxiliary: 0.2765 | Cardinality: 0.1397 | Classification: 0.0984 | Classification_binary: 0.1927 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1867 | Regression_eta: 0.0017 | Regression_pt: 0.1869 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0941 | Regression_eta: 0.0528 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0529 | LR: 4.19e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1833s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0147 | max_forward: 0.6791 | max_loss: 0.1546 | max_backward: 1.3392 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.9294 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7463.4 | uss_MB: 7147.9 | child_rss_MB: 4672.2 | allocated_MB: 345.2 | reserved_MB: 25564.0 | inactive_split_MB: 221.3 +rank7 [2026-10-29 08:17:24,690] INFO : Step 29800/50000 rank7 | Train Loss: 1.5204 | Train Loss Unweighted: 0.7810 | Train Loss Components: Auxiliary: 0.3188 | Cardinality: 0.1856 | Classification: 0.1227 | Classification_binary: 0.2452 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2025 | Regression_eta: 0.0022 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2469 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0996 | Regression_eta: 0.0555 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0718 | LR: 4.16e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3852s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7465 | max_loss: 0.1466 | max_backward: 1.4183 | max_optimizer_step: 0.3010 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7460.8 | uss_MB: 7146.3 | child_rss_MB: 4705.1 | allocated_MB: 348.1 | reserved_MB: 25564.0 | inactive_split_MB: 237.9 +rank7 [2026-10-29 08:21:10,554] INFO : Step 29900/50000 rank7 | Train Loss: 1.4604 | Train Loss Unweighted: 0.7522 | Train Loss Components: Auxiliary: 0.3102 | Cardinality: 0.1829 | Classification: 0.1007 | Classification_binary: 0.2214 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2116 | Regression_eta: 0.0020 | Regression_pt: 0.2143 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2224 | Regression_cos_phi: 0.1119 | Regression_energy: 0.1162 | Regression_eta: 0.0736 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0866 | LR: 4.14e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9961s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7725 | max_loss: 0.2249 | max_backward: 1.6977 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1623 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1430 | Memory: rss_MB: 7458.4 | uss_MB: 6704.5 | child_rss_MB: 4700.5 | allocated_MB: 346.8 | reserved_MB: 25564.0 | inactive_split_MB: 265.2 +rank7 [2026-10-29 08:24:56,386] INFO : Step 30000/50000 rank7 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6729 | Train Loss Components: Auxiliary: 0.2776 | Cardinality: 0.1319 | Classification: 0.0949 | Classification_binary: 0.1852 | HitGrouping: 0.2402 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1935 | Regression_eta: 0.0018 | Regression_pt: 0.1953 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0891 | Regression_eta: 0.0529 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0905 | LR: 4.11e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7041s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.6833 | max_loss: 0.2384 | max_backward: 1.6387 | max_optimizer_step: 0.2291 | max_tensorboard_logging: 1.1205 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7457.4 | uss_MB: 7144.1 | child_rss_MB: 4694.0 | allocated_MB: 343.0 | reserved_MB: 25564.0 | inactive_split_MB: 225.2 +rank7 [2026-10-29 08:24:57,340] INFO : Running validation on rank7 +rank7 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 08:24:57,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 08:25:45,090] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:25:45,653] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 08:26:11,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 08:26:12,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 08:26:13,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 08:26:14,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 08:26:24,218] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:26:24,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:26:25,382] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 08:26:41,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 08:26:42,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 08:26:43,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 08:26:44,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 08:26:54,237] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:26:54,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:26:55,053] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 08:27:10,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 08:27:11,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 08:27:11,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 08:27:12,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_30000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 08:27:22,303] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 08:31:31,211] INFO : Step 30100/50000 rank7 | Train Loss: 1.4244 | Train Loss Unweighted: 0.7244 | Train Loss Components: Auxiliary: 0.3080 | Cardinality: 0.1608 | Classification: 0.1064 | Classification_binary: 0.2200 | HitGrouping: 0.2393 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1964 | Regression_eta: 0.0021 | Regression_pt: 0.1973 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0947 | Regression_eta: 0.0564 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0638 | LR: 4.08e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8523s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8562 | max_loss: 0.1528 | max_backward: 1.3684 | max_optimizer_step: 0.2620 | max_tensorboard_logging: 1.2574 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 7536.9 | uss_MB: 7214.1 | child_rss_MB: 4666.1 | allocated_MB: 346.8 | reserved_MB: 23488.0 | inactive_split_MB: 235.2 +rank7 [2026-10-29 08:35:23,904] INFO : Step 30200/50000 rank7 | Train Loss: 1.4388 | Train Loss Unweighted: 0.7419 | Train Loss Components: Auxiliary: 0.3112 | Cardinality: 0.1625 | Classification: 0.0999 | Classification_binary: 0.2279 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2046 | Regression_eta: 0.0018 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.2226 | Regression_cos_phi: 0.1010 | Regression_energy: 0.0900 | Regression_eta: 0.0609 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0905 | LR: 4.06e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5320s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7612 | max_loss: 0.1495 | max_backward: 1.7285 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7530 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0565 | Memory: rss_MB: 7543.8 | uss_MB: 7222.0 | child_rss_MB: 4689.5 | allocated_MB: 344.3 | reserved_MB: 23488.0 | inactive_split_MB: 229.9 +rank7 [2026-10-29 08:39:30,452] INFO : Step 30300/50000 rank7 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1325 | Classification: 0.0841 | Classification_binary: 0.1881 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1918 | Regression_eta: 0.0019 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1976 | Regression_cos_phi: 0.1138 | Regression_energy: 0.0913 | Regression_eta: 0.0865 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0907 | LR: 4.03e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.1811s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0417 | max_forward: 0.7079 | max_loss: 0.1717 | max_backward: 1.4657 | max_optimizer_step: 0.3179 | max_tensorboard_logging: 0.6064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7543.3 | uss_MB: 7222.2 | child_rss_MB: 4688.3 | allocated_MB: 345.0 | reserved_MB: 25492.0 | inactive_split_MB: 227.0 +rank7 [2026-10-29 08:43:19,168] INFO : Step 30400/50000 rank7 | Train Loss: 1.3580 | Train Loss Unweighted: 0.7292 | Train Loss Components: Auxiliary: 0.2899 | Cardinality: 0.1293 | Classification: 0.1143 | Classification_binary: 0.1997 | HitGrouping: 0.2357 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2051 | Regression_eta: 0.0020 | Regression_pt: 0.2060 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0925 | Regression_eta: 0.0635 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0709 | LR: 4.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5063s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7304 | max_loss: 0.1498 | max_backward: 1.7289 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7814 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7543.7 | uss_MB: 7223.3 | child_rss_MB: 4661.4 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 240.9 +rank7 [2026-10-29 08:47:03,580] INFO : Step 30500/50000 rank7 | Train Loss: 1.3932 | Train Loss Unweighted: 0.7118 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1521 | Classification: 0.1068 | Classification_binary: 0.2247 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0020 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1961 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0830 | Regression_eta: 0.0660 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0571 | LR: 3.98e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7816 | max_loss: 0.2578 | max_backward: 1.8850 | max_optimizer_step: 0.2360 | max_tensorboard_logging: 0.8091 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7544.9 | uss_MB: 7224.1 | child_rss_MB: 4657.0 | allocated_MB: 345.3 | reserved_MB: 25492.0 | inactive_split_MB: 280.7 +rank7 [2026-10-29 08:50:50,677] INFO : Step 30600/50000 rank7 | Train Loss: 1.3017 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2736 | Cardinality: 0.1453 | Classification: 0.0934 | Classification_binary: 0.2026 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1760 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0763 | Regression_eta: 0.0601 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0599 | LR: 3.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5464s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7562 | max_loss: 0.1588 | max_backward: 1.3239 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7542.9 | uss_MB: 7222.2 | child_rss_MB: 4688.0 | allocated_MB: 342.6 | reserved_MB: 25492.0 | inactive_split_MB: 256.6 +rank7 [2026-10-29 08:54:35,589] INFO : Step 30700/50000 rank7 | Train Loss: 1.2565 | Train Loss Unweighted: 0.6423 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1070 | Classification: 0.0918 | Classification_binary: 0.1844 | HitGrouping: 0.2345 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1797 | Regression_eta: 0.0019 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2312 | Regression_cos_phi: 0.1396 | Regression_energy: 0.0896 | Regression_eta: 0.0844 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0944 | LR: 3.92e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2957s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.1447 | max_backward: 1.2369 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 1.0242 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0718 | Memory: rss_MB: 7545.8 | uss_MB: 7226.1 | child_rss_MB: 4665.5 | allocated_MB: 346.8 | reserved_MB: 25492.0 | inactive_split_MB: 240.0 +rank7 [2026-10-29 08:58:20,945] INFO : Step 30800/50000 rank7 | Train Loss: 1.4939 | Train Loss Unweighted: 0.7667 | Train Loss Components: Auxiliary: 0.3252 | Cardinality: 0.1705 | Classification: 0.1145 | Classification_binary: 0.2362 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2048 | Regression_eta: 0.0020 | Regression_pt: 0.2067 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0991 | Regression_eta: 0.0631 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0889 | LR: 3.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2068s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.7262 | max_loss: 0.1582 | max_backward: 1.4447 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 1.5825 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1456 | Memory: rss_MB: 7547.4 | uss_MB: 7228.5 | child_rss_MB: 4662.4 | allocated_MB: 346.1 | reserved_MB: 25492.0 | inactive_split_MB: 193.7 +rank7 [2026-10-29 09:02:09,797] INFO : Step 30900/50000 rank7 | Train Loss: 1.3312 | Train Loss Unweighted: 0.6882 | Train Loss Components: Auxiliary: 0.2877 | Cardinality: 0.1396 | Classification: 0.0971 | Classification_binary: 0.2170 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1843 | Regression_eta: 0.0018 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2154 | Regression_cos_phi: 0.1020 | Regression_energy: 0.1047 | Regression_eta: 0.0599 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0878 | LR: 3.87e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9283s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0437 | max_forward: 0.7790 | max_loss: 0.1480 | max_backward: 1.4594 | max_optimizer_step: 0.2667 | max_tensorboard_logging: 1.0518 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 7545.6 | uss_MB: 6322.5 | child_rss_MB: 4691.4 | allocated_MB: 341.3 | reserved_MB: 25492.0 | inactive_split_MB: 170.5 +rank7 [2026-10-29 09:05:52,365] INFO : Step 31000/50000 rank7 | Train Loss: 1.1995 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2548 | Cardinality: 0.1343 | Classification: 0.0934 | Classification_binary: 0.1668 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1728 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0850 | Regression_eta: 0.0544 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0809 | LR: 3.84e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.1876s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1113 | max_forward: 0.7323 | max_loss: 0.1436 | max_backward: 1.1237 | max_optimizer_step: 0.2378 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0810 | Memory: rss_MB: 7546.1 | uss_MB: 7226.7 | child_rss_MB: 4691.6 | allocated_MB: 344.1 | reserved_MB: 25492.0 | inactive_split_MB: 246.5 +rank7 [2026-10-29 09:05:53,358] INFO : Running validation on rank7 +rank7 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:05:53,358] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:06:41,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:06:42,226] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 09:07:08,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:07:09,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:07:10,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:07:11,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:07:21,366] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 09:07:22,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:07:22,581] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 09:07:38,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:07:39,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:07:40,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:07:41,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:07:51,316] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 09:07:51,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:07:52,509] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 09:08:08,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:08:08,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:08:09,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:08:10,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_31000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:08:20,204] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 09:12:27,860] INFO : Step 31100/50000 rank7 | Train Loss: 1.4116 | Train Loss Unweighted: 0.6983 | Train Loss Components: Auxiliary: 0.2979 | Cardinality: 0.1586 | Classification: 0.1058 | Classification_binary: 0.2214 | HitGrouping: 0.2334 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1817 | Regression_eta: 0.0022 | Regression_pt: 0.1849 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2407 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0785 | Regression_eta: 0.0794 | Regression_pt: 0.0821 | Regression_sin_phi: 0.1216 | LR: 3.82e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4622s | Phase Times: max_pre_step_barrier: 0.1110 | max_batch_to_device: 0.0117 | max_forward: 0.9272 | max_loss: 0.1696 | max_backward: 2.2116 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.9457 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 9188.4 | uss_MB: 7759.4 | child_rss_MB: 4700.9 | allocated_MB: 347.3 | reserved_MB: 23048.0 | inactive_split_MB: 260.5 +rank7 [2026-10-29 09:16:29,206] INFO : Step 31200/50000 rank7 | Train Loss: 1.4320 | Train Loss Unweighted: 0.7320 | Train Loss Components: Auxiliary: 0.3124 | Cardinality: 0.1693 | Classification: 0.1071 | Classification_binary: 0.2448 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1868 | Regression_eta: 0.0019 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2238 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0960 | Regression_eta: 0.0488 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0649 | LR: 3.79e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5354s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8930 | max_loss: 0.1519 | max_backward: 1.7171 | max_optimizer_step: 0.2419 | max_tensorboard_logging: 0.6094 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0589 | Memory: rss_MB: 7661.1 | uss_MB: 7341.6 | child_rss_MB: 4699.3 | allocated_MB: 342.9 | reserved_MB: 23204.0 | inactive_split_MB: 184.0 +rank7 [2026-10-29 09:20:24,830] INFO : Step 31300/50000 rank7 | Train Loss: 1.2588 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1618 | Classification: 0.0847 | Classification_binary: 0.1787 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1725 | Regression_eta: 0.0019 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0769 | Regression_eta: 0.0601 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0698 | LR: 3.76e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.4675s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 2.4774 | max_loss: 0.5633 | max_backward: 1.9541 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.5140 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7657.6 | uss_MB: 7341.9 | child_rss_MB: 4687.3 | allocated_MB: 343.7 | reserved_MB: 23204.0 | inactive_split_MB: 239.1 +rank7 [2026-10-29 09:24:20,382] INFO : Step 31400/50000 rank7 | Train Loss: 1.2937 | Train Loss Unweighted: 0.6467 | Train Loss Components: Auxiliary: 0.2720 | Cardinality: 0.1434 | Classification: 0.0992 | Classification_binary: 0.1885 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1765 | Regression_eta: 0.0020 | Regression_pt: 0.1784 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0881 | Regression_eta: 0.0713 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0540 | LR: 3.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8376s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7547 | max_loss: 0.2612 | max_backward: 1.8914 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7647.8 | uss_MB: 7342.9 | child_rss_MB: 4638.3 | allocated_MB: 344.2 | reserved_MB: 23204.0 | inactive_split_MB: 255.6 +rank7 [2026-10-29 09:28:17,181] INFO : Step 31500/50000 rank7 | Train Loss: 1.4530 | Train Loss Unweighted: 0.7283 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.1760 | Classification: 0.1073 | Classification_binary: 0.2284 | HitGrouping: 0.2354 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1934 | Regression_eta: 0.0023 | Regression_pt: 0.1946 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2464 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0902 | Regression_eta: 0.0632 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0629 | LR: 3.71e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.3642s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1155 | max_forward: 1.4975 | max_loss: 0.1892 | max_backward: 2.2187 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.8105 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1354 | Memory: rss_MB: 7643.1 | uss_MB: 7218.2 | child_rss_MB: 4606.8 | allocated_MB: 344.1 | reserved_MB: 24464.0 | inactive_split_MB: 213.4 +rank7 [2026-10-29 09:32:16,798] INFO : Step 31600/50000 rank7 | Train Loss: 1.2636 | Train Loss Unweighted: 0.6340 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1359 | Classification: 0.0906 | Classification_binary: 0.1911 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1744 | Regression_eta: 0.0018 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0968 | Regression_eta: 0.0473 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0769 | LR: 3.69e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.8619s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7027 | max_loss: 0.5449 | max_backward: 1.9216 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.9105 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7636.8 | uss_MB: 7343.7 | child_rss_MB: 4626.2 | allocated_MB: 347.4 | reserved_MB: 24464.0 | inactive_split_MB: 237.1 +rank7 [2026-10-29 09:36:09,297] INFO : Step 31700/50000 rank7 | Train Loss: 1.1972 | Train Loss Unweighted: 0.6103 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1145 | Classification: 0.0977 | Classification_binary: 0.1713 | HitGrouping: 0.2306 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1686 | Regression_eta: 0.0017 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0931 | Regression_eta: 0.0575 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0637 | LR: 3.66e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1352s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 1.0231 | max_loss: 0.2590 | max_backward: 1.0972 | max_optimizer_step: 0.2361 | max_tensorboard_logging: 0.8870 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 7630.3 | uss_MB: 7348.9 | child_rss_MB: 4583.5 | allocated_MB: 341.2 | reserved_MB: 24464.0 | inactive_split_MB: 232.8 +rank7 [2026-10-29 09:40:22,623] INFO : Step 31800/50000 rank7 | Train Loss: 1.3629 | Train Loss Unweighted: 0.6614 | Train Loss Components: Auxiliary: 0.2927 | Cardinality: 0.1758 | Classification: 0.1057 | Classification_binary: 0.2166 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1662 | Regression_eta: 0.0020 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2362 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0836 | Regression_eta: 0.0592 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0770 | LR: 3.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3539s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7238 | max_loss: 0.1473 | max_backward: 1.3627 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.9253 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7632.3 | uss_MB: 7349.0 | child_rss_MB: 4552.7 | allocated_MB: 345.0 | reserved_MB: 24464.0 | inactive_split_MB: 223.1 +rank7 [2026-10-29 09:48:15,344] INFO : Step 31900/50000 rank7 | Train Loss: 1.2579 | Train Loss Unweighted: 0.6392 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1370 | Classification: 0.0927 | Classification_binary: 0.2103 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0016 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2141 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0817 | Regression_eta: 0.0564 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0763 | LR: 3.61e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8685s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7474 | max_loss: 0.1567 | max_backward: 1.7749 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.9592 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7629.7 | uss_MB: 6872.4 | child_rss_MB: 4567.3 | allocated_MB: 345.4 | reserved_MB: 24464.0 | inactive_split_MB: 296.5 +rank7 [2026-10-29 09:52:25,845] INFO : Step 32000/50000 rank7 | Train Loss: 1.2067 | Train Loss Unweighted: 0.6180 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1214 | Classification: 0.0890 | Classification_binary: 0.1713 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1768 | Regression_eta: 0.0017 | Regression_pt: 0.1773 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0834 | Regression_eta: 0.0402 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0790 | LR: 3.58e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.1231s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7689 | max_loss: 0.1474 | max_backward: 1.4521 | max_optimizer_step: 0.2391 | max_tensorboard_logging: 0.5994 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0734 | Memory: rss_MB: 7611.4 | uss_MB: 7343.9 | child_rss_MB: 4547.8 | allocated_MB: 344.2 | reserved_MB: 24466.0 | inactive_split_MB: 266.4 +rank7 [2026-10-29 09:52:26,792] INFO : Running validation on rank7 +rank7 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:52:26,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:53:17,424] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:53:17,942] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 09:53:45,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:53:46,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:53:48,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:53:49,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:53:59,278] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 09:54:00,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:54:01,009] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 09:54:16,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:54:17,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:54:18,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:54:19,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:54:29,668] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 09:54:30,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:54:30,789] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 09:54:45,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:54:46,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:54:47,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:54:47,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_32000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:54:57,825] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 09:59:19,241] INFO : Step 32100/50000 rank7 | Train Loss: 1.2426 | Train Loss Unweighted: 0.6546 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1233 | Classification: 0.1000 | Classification_binary: 0.1824 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1829 | Regression_eta: 0.0017 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0793 | Regression_eta: 0.0586 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0613 | LR: 3.56e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9459s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7418 | max_loss: 0.1464 | max_backward: 1.7604 | max_optimizer_step: 0.2636 | max_tensorboard_logging: 1.0777 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0964 | Memory: rss_MB: 9219.8 | uss_MB: 8948.0 | child_rss_MB: 4539.4 | allocated_MB: 346.4 | reserved_MB: 23472.0 | inactive_split_MB: 248.5 +rank7 [2026-10-29 10:03:26,458] INFO : Step 32200/50000 rank7 | Train Loss: 1.3781 | Train Loss Unweighted: 0.6855 | Train Loss Components: Auxiliary: 0.2960 | Cardinality: 0.1706 | Classification: 0.1022 | Classification_binary: 0.2291 | HitGrouping: 0.2207 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1739 | Regression_eta: 0.0019 | Regression_pt: 0.1761 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2348 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0799 | Regression_eta: 0.0737 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0851 | LR: 3.53e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5229s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8249 | max_loss: 0.1535 | max_backward: 1.4726 | max_optimizer_step: 0.2387 | max_tensorboard_logging: 0.8800 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7675.3 | uss_MB: 7404.5 | child_rss_MB: 4522.0 | allocated_MB: 345.7 | reserved_MB: 24462.0 | inactive_split_MB: 306.1 +rank7 [2026-10-29 10:07:22,792] INFO : Step 32300/50000 rank7 | Train Loss: 1.2207 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1485 | Classification: 0.0834 | Classification_binary: 0.1764 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1611 | Regression_eta: 0.0018 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0739 | Regression_eta: 0.0734 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0765 | LR: 3.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5879s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.7074 | max_loss: 0.2203 | max_backward: 1.7102 | max_optimizer_step: 0.2349 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 7674.9 | uss_MB: 7407.3 | child_rss_MB: 4547.1 | allocated_MB: 342.2 | reserved_MB: 24462.0 | inactive_split_MB: 227.4 +rank7 [2026-10-29 10:11:42,985] INFO : Step 32400/50000 rank7 | Train Loss: 1.1824 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2483 | Cardinality: 0.1246 | Classification: 0.0940 | Classification_binary: 0.1731 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1623 | Regression_eta: 0.0018 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0763 | Regression_eta: 0.0287 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0515 | LR: 3.48e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0242s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6956 | max_loss: 0.2099 | max_backward: 1.2817 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.7936 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0579 | Memory: rss_MB: 7670.1 | uss_MB: 7404.5 | child_rss_MB: 4547.2 | allocated_MB: 344.3 | reserved_MB: 26474.0 | inactive_split_MB: 239.0 +rank7 [2026-10-29 10:15:56,354] INFO : Step 32500/50000 rank7 | Train Loss: 1.3206 | Train Loss Unweighted: 0.6869 | Train Loss Components: Auxiliary: 0.2858 | Cardinality: 0.1356 | Classification: 0.1067 | Classification_binary: 0.2115 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1814 | Regression_eta: 0.0018 | Regression_pt: 0.1834 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0880 | Regression_eta: 0.0628 | Regression_pt: 0.0920 | Regression_sin_phi: 0.0998 | LR: 3.46e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5245s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.8693 | max_loss: 0.2006 | max_backward: 2.2503 | max_optimizer_step: 0.3348 | max_tensorboard_logging: 0.8340 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7674.6 | uss_MB: 5750.4 | child_rss_MB: 4522.5 | allocated_MB: 346.9 | reserved_MB: 26474.0 | inactive_split_MB: 282.9 +rank7 [2026-10-29 10:20:13,267] INFO : Step 32600/50000 rank7 | Train Loss: 1.3294 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2821 | Cardinality: 0.1541 | Classification: 0.0902 | Classification_binary: 0.2137 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1733 | Regression_eta: 0.0018 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0855 | Regression_eta: 0.0762 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0850 | LR: 3.43e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4772s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7604 | max_loss: 0.1452 | max_backward: 1.4970 | max_optimizer_step: 0.3232 | max_tensorboard_logging: 0.8000 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0560 | Memory: rss_MB: 7674.5 | uss_MB: 7409.0 | child_rss_MB: 4559.0 | allocated_MB: 342.2 | reserved_MB: 27526.0 | inactive_split_MB: 247.1 +rank7 [2026-10-29 10:24:11,750] INFO : Step 32700/50000 rank7 | Train Loss: 1.2190 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2605 | Cardinality: 0.1281 | Classification: 0.0838 | Classification_binary: 0.1760 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1704 | Regression_eta: 0.0017 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0748 | Regression_eta: 0.0895 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0758 | LR: 3.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0504s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6957 | max_loss: 0.1406 | max_backward: 1.2688 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7404 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 7675.5 | uss_MB: 7412.1 | child_rss_MB: 4532.2 | allocated_MB: 344.6 | reserved_MB: 27526.0 | inactive_split_MB: 241.0 +rank7 [2026-10-29 10:28:05,808] INFO : Step 32800/50000 rank7 | Train Loss: 1.2959 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2763 | Cardinality: 0.1373 | Classification: 0.1136 | Classification_binary: 0.2115 | HitGrouping: 0.2305 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1634 | Regression_eta: 0.0018 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0728 | Regression_eta: 0.0570 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0548 | LR: 3.38e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5515s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7490 | max_loss: 0.2229 | max_backward: 1.5897 | max_optimizer_step: 0.2402 | max_tensorboard_logging: 0.9255 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7669.2 | uss_MB: 7405.8 | child_rss_MB: 4538.8 | allocated_MB: 345.2 | reserved_MB: 27526.0 | inactive_split_MB: 211.2 +rank7 [2026-10-29 10:32:14,147] INFO : Step 32900/50000 rank7 | Train Loss: 1.3941 | Train Loss Unweighted: 0.6817 | Train Loss Components: Auxiliary: 0.2945 | Cardinality: 0.1783 | Classification: 0.1039 | Classification_binary: 0.2219 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0019 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2353 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0785 | Regression_eta: 0.0567 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0654 | LR: 3.36e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.8073s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8226 | max_loss: 0.1551 | max_backward: 1.5905 | max_optimizer_step: 0.3306 | max_tensorboard_logging: 0.7838 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7678.5 | uss_MB: 6939.8 | child_rss_MB: 4557.8 | allocated_MB: 346.4 | reserved_MB: 27526.0 | inactive_split_MB: 253.0 +rank7 [2026-10-29 10:36:14,933] INFO : Step 33000/50000 rank7 | Train Loss: 1.2030 | Train Loss Unweighted: 0.5934 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1365 | Classification: 0.0832 | Classification_binary: 0.1650 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0016 | Regression_pt: 0.1710 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0848 | Regression_eta: 0.0570 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0448 | LR: 3.33e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7648s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6842 | max_loss: 0.1420 | max_backward: 1.1829 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5228 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1223 | Memory: rss_MB: 7678.2 | uss_MB: 7410.5 | child_rss_MB: 4565.0 | allocated_MB: 341.2 | reserved_MB: 27526.0 | inactive_split_MB: 229.0 +rank7 [2026-10-29 10:36:15,962] INFO : Running validation on rank7 +rank7 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 10:36:15,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 10:37:03,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:37:04,372] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 10:37:32,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 10:37:33,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 10:37:35,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 10:37:37,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 10:37:47,501] INFO : Time taken to make predictions on device 7 is: 0.30 min +rank7 [2026-10-29 10:37:48,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:37:48,816] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 10:38:05,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 10:38:06,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 10:38:07,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 10:38:08,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 10:38:18,298] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:38:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:38:21,705] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 10:38:37,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 10:38:37,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 10:38:38,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 10:38:39,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_33000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 10:38:49,101] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 10:43:12,058] INFO : Step 33100/50000 rank7 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5941 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1309 | Classification: 0.0899 | Classification_binary: 0.1792 | HitGrouping: 0.2222 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1602 | Regression_eta: 0.0015 | Regression_pt: 0.1616 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0492 | Regression_energy: 0.1003 | Regression_eta: 0.0450 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0659 | LR: 3.31e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.9490s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0122 | max_forward: 1.0189 | max_loss: 0.4588 | max_backward: 1.9573 | max_optimizer_step: 0.2668 | max_tensorboard_logging: 0.7265 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0847 | Memory: rss_MB: 9299.4 | uss_MB: 9029.8 | child_rss_MB: 4536.9 | allocated_MB: 342.0 | reserved_MB: 23444.0 | inactive_split_MB: 236.0 +rank7 [2026-10-29 10:47:22,843] INFO : Step 33200/50000 rank7 | Train Loss: 1.3463 | Train Loss Unweighted: 0.6685 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1543 | Classification: 0.1022 | Classification_binary: 0.2232 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1683 | Regression_eta: 0.0021 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2290 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0676 | Regression_eta: 0.0813 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0891 | LR: 3.28e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6971s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7687 | max_loss: 0.1598 | max_backward: 1.6817 | max_optimizer_step: 0.3495 | max_tensorboard_logging: 0.7720 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7719.5 | uss_MB: 7450.8 | child_rss_MB: 4547.7 | allocated_MB: 345.6 | reserved_MB: 24438.0 | inactive_split_MB: 183.7 +rank7 [2026-10-29 10:51:33,085] INFO : Step 33300/50000 rank7 | Train Loss: 1.1827 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1107 | Classification: 0.0881 | Classification_binary: 0.1830 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0786 | Regression_eta: 0.0597 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0637 | LR: 3.26e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4499s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0060 | max_forward: 0.8483 | max_loss: 0.1589 | max_backward: 1.3801 | max_optimizer_step: 0.2514 | max_tensorboard_logging: 0.8485 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 7720.5 | uss_MB: 7450.3 | child_rss_MB: 4547.4 | allocated_MB: 345.3 | reserved_MB: 24438.0 | inactive_split_MB: 227.0 +rank7 [2026-10-29 10:55:47,785] INFO : Step 33400/50000 rank7 | Train Loss: 1.1478 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2418 | Cardinality: 0.1004 | Classification: 0.0893 | Classification_binary: 0.1623 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1644 | Regression_eta: 0.0016 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0729 | Regression_eta: 0.0407 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0607 | LR: 3.23e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7041 | max_loss: 0.2113 | max_backward: 1.7088 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.8474 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0606 | Memory: rss_MB: 7717.9 | uss_MB: 7448.5 | child_rss_MB: 4538.4 | allocated_MB: 344.7 | reserved_MB: 25174.0 | inactive_split_MB: 256.9 +rank7 [2026-10-29 10:59:48,623] INFO : Step 33500/50000 rank7 | Train Loss: 1.3793 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2920 | Cardinality: 0.1599 | Classification: 0.1022 | Classification_binary: 0.2257 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1750 | Regression_eta: 0.0020 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.2370 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0796 | Regression_eta: 0.0649 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0794 | LR: 3.21e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.9167s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1045 | max_forward: 2.4147 | max_loss: 0.3508 | max_backward: 2.3324 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.8391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 7719.9 | uss_MB: 7452.3 | child_rss_MB: 4557.9 | allocated_MB: 346.0 | reserved_MB: 25434.0 | inactive_split_MB: 198.2 +rank7 [2026-10-29 11:05:24,497] INFO : Step 33600/50000 rank7 | Train Loss: 1.3232 | Train Loss Unweighted: 0.6733 | Train Loss Components: Auxiliary: 0.2848 | Cardinality: 0.1351 | Classification: 0.1017 | Classification_binary: 0.2153 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1753 | Regression_eta: 0.0019 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0750 | Regression_eta: 0.0805 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0800 | LR: 3.19e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.2013s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 1.0081 | max_loss: 0.2197 | max_backward: 2.0533 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.8457 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 7714.6 | uss_MB: 7447.3 | child_rss_MB: 4537.5 | allocated_MB: 344.4 | reserved_MB: 26198.0 | inactive_split_MB: 275.7 +rank7 [2026-10-29 11:09:08,678] INFO : Step 33700/50000 rank7 | Train Loss: 1.1862 | Train Loss Unweighted: 0.5824 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1341 | Classification: 0.0878 | Classification_binary: 0.1688 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1615 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0770 | Regression_eta: 0.0608 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0858 | LR: 3.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3162s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6557 | max_loss: 0.1415 | max_backward: 1.1831 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 1.1415 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 7720.6 | uss_MB: 7451.6 | child_rss_MB: 4536.3 | allocated_MB: 341.4 | reserved_MB: 26198.0 | inactive_split_MB: 239.6 +rank7 [2026-10-29 11:13:02,287] INFO : Step 33800/50000 rank7 | Train Loss: 1.2788 | Train Loss Unweighted: 0.6639 | Train Loss Components: Auxiliary: 0.2760 | Cardinality: 0.1188 | Classification: 0.1083 | Classification_binary: 0.2043 | HitGrouping: 0.2275 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0019 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0720 | Regression_eta: 0.0838 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0994 | LR: 3.14e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3675s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7333 | max_loss: 0.1465 | max_backward: 1.3435 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.9196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7715.9 | uss_MB: 7449.0 | child_rss_MB: 4540.2 | allocated_MB: 346.7 | reserved_MB: 27226.0 | inactive_split_MB: 245.8 +rank7 [2026-10-29 11:16:52,541] INFO : Step 33900/50000 rank7 | Train Loss: 1.2942 | Train Loss Unweighted: 0.6453 | Train Loss Components: Auxiliary: 0.2814 | Cardinality: 0.1369 | Classification: 0.0896 | Classification_binary: 0.2172 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0018 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0905 | Regression_eta: 0.0577 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0882 | LR: 3.11e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8122s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0060 | max_forward: 0.8432 | max_loss: 0.2675 | max_backward: 1.6780 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8487 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 7719.5 | uss_MB: 7452.7 | child_rss_MB: 4538.0 | allocated_MB: 344.7 | reserved_MB: 27226.0 | inactive_split_MB: 282.0 +rank7 [2026-10-29 11:20:47,786] INFO : Step 34000/50000 rank7 | Train Loss: 1.1988 | Train Loss Unweighted: 0.5891 | Train Loss Components: Auxiliary: 0.2479 | Cardinality: 0.1319 | Classification: 0.0825 | Classification_binary: 0.1681 | HitGrouping: 0.2223 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1663 | Regression_eta: 0.0018 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0864 | Regression_eta: 0.0524 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0787 | LR: 3.09e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9498s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0644 | max_forward: 0.7590 | max_loss: 0.1462 | max_backward: 1.4332 | max_optimizer_step: 0.3438 | max_tensorboard_logging: 0.9353 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7721.5 | uss_MB: 7453.0 | child_rss_MB: 4539.2 | allocated_MB: 342.1 | reserved_MB: 27226.0 | inactive_split_MB: 272.5 +rank7 [2026-10-29 11:20:48,852] INFO : Running validation on rank7 +rank7 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 11:20:48,852] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 11:21:42,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:21:42,910] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 11:22:09,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 11:22:10,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 11:22:11,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 11:22:12,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 11:22:22,031] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 11:22:22,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:22:22,978] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 11:22:39,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 11:22:40,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 11:22:41,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 11:22:42,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 11:22:52,608] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 11:22:52,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:22:53,478] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 11:23:09,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 11:23:09,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 11:23:10,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 11:23:11,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_34000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 11:23:21,022] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 11:27:42,516] INFO : Step 34100/50000 rank7 | Train Loss: 1.2543 | Train Loss Unweighted: 0.6200 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1449 | Classification: 0.0973 | Classification_binary: 0.1865 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0782 | Regression_eta: 0.0478 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0822 | LR: 3.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 7.5173s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0068 | max_forward: 0.7753 | max_loss: 0.1416 | max_backward: 1.5339 | max_optimizer_step: 0.2612 | max_tensorboard_logging: 3.8008 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0763 | Memory: rss_MB: 9326.3 | uss_MB: 9062.7 | child_rss_MB: 4535.4 | allocated_MB: 347.9 | reserved_MB: 23094.0 | inactive_split_MB: 240.0 +rank7 [2026-10-29 11:31:45,286] INFO : Step 34200/50000 rank7 | Train Loss: 1.3478 | Train Loss Unweighted: 0.6536 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1681 | Classification: 0.1006 | Classification_binary: 0.2198 | HitGrouping: 0.2235 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1638 | Regression_eta: 0.0019 | Regression_pt: 0.1653 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0777 | Regression_eta: 0.0598 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0778 | LR: 3.04e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7882s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0066 | max_forward: 1.1820 | max_loss: 0.2331 | max_backward: 2.0074 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5138 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1348 | Memory: rss_MB: 7835.7 | uss_MB: 7573.5 | child_rss_MB: 4531.7 | allocated_MB: 345.0 | reserved_MB: 23136.0 | inactive_split_MB: 263.6 +rank7 [2026-10-29 11:35:56,849] INFO : Step 34300/50000 rank7 | Train Loss: 1.2345 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2644 | Cardinality: 0.1296 | Classification: 0.0879 | Classification_binary: 0.1981 | HitGrouping: 0.2187 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1651 | Regression_eta: 0.0016 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0847 | Regression_eta: 0.0624 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0756 | LR: 3.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 7.2778s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7567 | max_loss: 0.1469 | max_backward: 2.9251 | max_optimizer_step: 0.3848 | max_tensorboard_logging: 0.7946 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.0612 | Memory: rss_MB: 7837.7 | uss_MB: 5782.8 | child_rss_MB: 4557.8 | allocated_MB: 341.6 | reserved_MB: 24096.0 | inactive_split_MB: 244.8 +rank7 [2026-10-29 11:40:08,369] INFO : Step 34400/50000 rank7 | Train Loss: 1.2059 | Train Loss Unweighted: 0.6101 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1152 | Classification: 0.0979 | Classification_binary: 0.1612 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1728 | Regression_eta: 0.0016 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0725 | Regression_eta: 0.0716 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0873 | LR: 2.99e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.9076s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6165 | max_loss: 0.1898 | max_backward: 2.2768 | max_optimizer_step: 0.2905 | max_tensorboard_logging: 1.0194 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.1 | uss_MB: 4906.3 | child_rss_MB: 4553.7 | allocated_MB: 344.7 | reserved_MB: 24096.0 | inactive_split_MB: 220.4 +rank7 [2026-10-29 11:44:04,819] INFO : Step 34500/50000 rank7 | Train Loss: 1.3186 | Train Loss Unweighted: 0.6767 | Train Loss Components: Auxiliary: 0.2845 | Cardinality: 0.1324 | Classification: 0.0981 | Classification_binary: 0.2189 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1771 | Regression_eta: 0.0018 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2082 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0810 | Regression_eta: 0.0660 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0658 | LR: 2.97e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5067s | Phase Times: max_pre_step_barrier: 0.1085 | max_batch_to_device: 0.0075 | max_forward: 0.8062 | max_loss: 0.2461 | max_backward: 1.7641 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.5238 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7840.1 | uss_MB: 7578.5 | child_rss_MB: 4549.4 | allocated_MB: 345.3 | reserved_MB: 25118.0 | inactive_split_MB: 267.9 +rank7 [2026-10-29 11:47:58,872] INFO : Step 34600/50000 rank7 | Train Loss: 1.3356 | Train Loss Unweighted: 0.6479 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1615 | Classification: 0.1003 | Classification_binary: 0.2230 | HitGrouping: 0.2287 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1595 | Regression_eta: 0.0020 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0781 | Regression_eta: 0.0615 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0910 | LR: 2.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6062s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7862 | max_loss: 0.1711 | max_backward: 1.7126 | max_optimizer_step: 0.2564 | max_tensorboard_logging: 0.6954 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0619 | Memory: rss_MB: 7833.6 | uss_MB: 7573.1 | child_rss_MB: 4544.4 | allocated_MB: 344.8 | reserved_MB: 25118.0 | inactive_split_MB: 257.4 +rank7 [2026-10-29 11:52:00,072] INFO : Step 34700/50000 rank7 | Train Loss: 1.1554 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1130 | Classification: 0.0873 | Classification_binary: 0.1684 | HitGrouping: 0.2254 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1585 | Regression_eta: 0.0016 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0755 | Regression_eta: 0.0514 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0793 | LR: 2.92e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.5596s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0180 | max_forward: 0.6801 | max_loss: 0.1403 | max_backward: 1.4206 | max_optimizer_step: 0.3544 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7835.6 | uss_MB: 6542.2 | child_rss_MB: 4551.0 | allocated_MB: 344.7 | reserved_MB: 25118.0 | inactive_split_MB: 227.8 +rank7 [2026-10-29 11:55:53,803] INFO : Step 34800/50000 rank7 | Train Loss: 1.2549 | Train Loss Unweighted: 0.6295 | Train Loss Components: Auxiliary: 0.2694 | Cardinality: 0.1335 | Classification: 0.1062 | Classification_binary: 0.1857 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1667 | Regression_eta: 0.0016 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2060 | Regression_cos_phi: 0.1158 | Regression_energy: 0.0861 | Regression_eta: 0.0815 | Regression_pt: 0.0913 | Regression_sin_phi: 0.1023 | LR: 2.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8590s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7279 | max_loss: 0.1458 | max_backward: 1.3328 | max_optimizer_step: 0.2432 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7836.6 | uss_MB: 7576.8 | child_rss_MB: 4548.3 | allocated_MB: 348.1 | reserved_MB: 25118.0 | inactive_split_MB: 190.1 +rank7 [2026-10-29 11:59:58,709] INFO : Step 34900/50000 rank7 | Train Loss: 1.2927 | Train Loss Unweighted: 0.6558 | Train Loss Components: Auxiliary: 0.2803 | Cardinality: 0.1482 | Classification: 0.0941 | Classification_binary: 0.2140 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1714 | Regression_eta: 0.0016 | Regression_pt: 0.1728 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2338 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0752 | Regression_eta: 0.0717 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0872 | LR: 2.88e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1714s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8300 | max_loss: 0.1562 | max_backward: 1.4287 | max_optimizer_step: 0.2431 | max_tensorboard_logging: 0.5329 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0711 | Memory: rss_MB: 7834.4 | uss_MB: 7575.9 | child_rss_MB: 4500.5 | allocated_MB: 346.4 | reserved_MB: 25118.0 | inactive_split_MB: 495.8 +rank7 [2026-10-29 12:04:06,317] INFO : Step 35000/50000 rank7 | Train Loss: 1.2809 | Train Loss Unweighted: 0.6303 | Train Loss Components: Auxiliary: 0.2711 | Cardinality: 0.1519 | Classification: 0.0923 | Classification_binary: 0.1882 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1722 | Regression_eta: 0.0017 | Regression_pt: 0.1738 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0812 | Regression_eta: 0.0615 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0704 | LR: 2.85e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4575s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7588 | max_loss: 0.1440 | max_backward: 1.6835 | max_optimizer_step: 0.2866 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7835.4 | uss_MB: 7582.9 | child_rss_MB: 4521.7 | allocated_MB: 344.1 | reserved_MB: 25118.0 | inactive_split_MB: 271.7 +rank7 [2026-10-29 12:04:07,348] INFO : Running validation on rank7 +rank7 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 12:04:07,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 12:05:04,843] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:05:05,390] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 12:05:31,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 12:05:32,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 12:05:33,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 12:05:34,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 12:05:44,782] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 12:05:45,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:05:45,911] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 12:06:02,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 12:06:02,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 12:06:03,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 12:06:04,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 12:06:14,861] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 12:06:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:06:16,077] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 12:06:31,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 12:06:32,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 12:06:32,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 12:06:33,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_35000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 12:06:43,613] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 12:11:06,965] INFO : Step 35100/50000 rank7 | Train Loss: 1.2678 | Train Loss Unweighted: 0.6412 | Train Loss Components: Auxiliary: 0.2734 | Cardinality: 0.1399 | Classification: 0.1122 | Classification_binary: 0.1677 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1776 | Regression_eta: 0.0018 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0895 | Regression_eta: 0.0647 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0888 | LR: 2.83e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.6538s | Phase Times: max_pre_step_barrier: 0.0414 | max_batch_to_device: 0.0133 | max_forward: 1.5567 | max_loss: 0.2479 | max_backward: 1.6259 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 1.2097 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0816 | Memory: rss_MB: 9328.3 | uss_MB: 9073.9 | child_rss_MB: 4527.8 | allocated_MB: 343.6 | reserved_MB: 23746.0 | inactive_split_MB: 227.1 +rank7 [2026-10-29 12:15:06,116] INFO : Step 35200/50000 rank7 | Train Loss: 1.2653 | Train Loss Unweighted: 0.6380 | Train Loss Components: Auxiliary: 0.2752 | Cardinality: 0.1302 | Classification: 0.0957 | Classification_binary: 0.2088 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1641 | Regression_eta: 0.0017 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0897 | Regression_eta: 0.0719 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0617 | LR: 2.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0570s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.7684 | max_loss: 0.1868 | max_backward: 1.6633 | max_optimizer_step: 0.2499 | max_tensorboard_logging: 0.8274 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 7908.5 | uss_MB: 5747.0 | child_rss_MB: 4531.8 | allocated_MB: 345.0 | reserved_MB: 24752.0 | inactive_split_MB: 193.7 +rank7 [2026-10-29 12:18:57,447] INFO : Step 35300/50000 rank7 | Train Loss: 1.2628 | Train Loss Unweighted: 0.6621 | Train Loss Components: Auxiliary: 0.2784 | Cardinality: 0.1367 | Classification: 0.0898 | Classification_binary: 0.1967 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1863 | Regression_eta: 0.0016 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0889 | Regression_eta: 0.0650 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0578 | LR: 2.79e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8177s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.2498 | max_backward: 1.7492 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0590 | Memory: rss_MB: 7909.7 | uss_MB: 7655.6 | child_rss_MB: 4525.3 | allocated_MB: 342.7 | reserved_MB: 24752.0 | inactive_split_MB: 196.2 +rank7 [2026-10-29 12:23:00,202] INFO : Step 35400/50000 rank7 | Train Loss: 1.1967 | Train Loss Unweighted: 0.6197 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1185 | Classification: 0.0868 | Classification_binary: 0.1579 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0018 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0930 | Regression_eta: 0.0472 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0531 | LR: 2.76e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3949s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0527 | max_forward: 0.7104 | max_loss: 0.1365 | max_backward: 1.4898 | max_optimizer_step: 0.2469 | max_tensorboard_logging: 0.9074 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0637 | Memory: rss_MB: 7901.4 | uss_MB: 7648.2 | child_rss_MB: 4527.6 | allocated_MB: 340.1 | reserved_MB: 24752.0 | inactive_split_MB: 232.0 +rank7 [2026-10-29 12:27:14,474] INFO : Step 35500/50000 rank7 | Train Loss: 1.2815 | Train Loss Unweighted: 0.6396 | Train Loss Components: Auxiliary: 0.2792 | Cardinality: 0.1510 | Classification: 0.0995 | Classification_binary: 0.1905 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1720 | Regression_eta: 0.0016 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0882 | Regression_eta: 0.0501 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0547 | LR: 2.74e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2676s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7753 | max_loss: 0.1658 | max_backward: 1.5353 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.6253 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7906.4 | uss_MB: 6064.5 | child_rss_MB: 4503.1 | allocated_MB: 344.1 | reserved_MB: 24752.0 | inactive_split_MB: 241.4 +rank7 [2026-10-29 12:31:17,345] INFO : Step 35600/50000 rank7 | Train Loss: 1.3358 | Train Loss Unweighted: 0.6656 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1499 | Classification: 0.0986 | Classification_binary: 0.2152 | HitGrouping: 0.2335 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1735 | Regression_eta: 0.0019 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2159 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0900 | Regression_eta: 0.0661 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0681 | LR: 2.72e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1620s | Phase Times: max_pre_step_barrier: 0.0144 | max_batch_to_device: 0.0742 | max_forward: 1.1316 | max_loss: 0.3177 | max_backward: 2.2251 | max_optimizer_step: 0.2467 | max_tensorboard_logging: 0.6418 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7905.4 | uss_MB: 7076.4 | child_rss_MB: 4512.2 | allocated_MB: 347.2 | reserved_MB: 24876.0 | inactive_split_MB: 297.0 +rank7 [2026-10-29 12:35:31,257] INFO : Step 35700/50000 rank7 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5869 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1445 | Classification: 0.0890 | Classification_binary: 0.1667 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1634 | Regression_eta: 0.0016 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0721 | Regression_eta: 0.0472 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0390 | LR: 2.70e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2778s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.6832 | max_loss: 0.1438 | max_backward: 1.1733 | max_optimizer_step: 0.2471 | max_tensorboard_logging: 0.9984 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7901.8 | uss_MB: 7654.0 | child_rss_MB: 4483.3 | allocated_MB: 342.5 | reserved_MB: 24876.0 | inactive_split_MB: 265.7 +rank7 [2026-10-29 14:49:26,313] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-29 14:49:26,317] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-29 14:49:53,253] INFO : Instantiating model +rank7 [2026-10-29 14:49:53,253] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-29 14:49:53,253] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-29 14:49:53,272] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-29 14:49:53,272] INFO : nn0 parameters: 171776 +rank7 [2026-10-29 14:49:53,272] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-29 14:49:53,273] INFO : Initializing backbone layer 0 +rank7 [2026-10-29 14:49:53,273] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,276] INFO : Initializing backbone layer 1 +rank7 [2026-10-29 14:49:53,276] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,279] INFO : Initializing backbone layer 2 +rank7 [2026-10-29 14:49:53,279] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,283] INFO : Initializing backbone layer 3 +rank7 [2026-10-29 14:49:53,283] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,286] INFO : Initializing backbone layer 4 +rank7 [2026-10-29 14:49:53,286] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,289] INFO : Initializing backbone layer 5 +rank7 [2026-10-29 14:49:53,289] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:49:53,292] INFO : Backbone initialization took 0.02s +rank7 [2026-10-29 14:49:53,292] INFO : backbone parameters: 2374656 +rank7 [2026-10-29 14:49:53,292] INFO : Initializing output DNNs +rank7 [2026-10-29 14:49:53,342] INFO : Output DNNs initialization took 0.05s +rank7 [2026-10-29 14:49:53,342] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-29 14:49:53,342] INFO : jagged_attention enabled=True +rank7 [2026-10-29 14:49:53,342] INFO : flash_attn_varlen enabled=True +rank7 [2026-10-29 14:49:53,342] INFO : task_queries enabled=False +rank7 [2026-10-29 14:49:53,342] INFO : classification_norm parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : regression_norm parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : classification_query parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : regression_query parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : classification_readout parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : regression_readout parameters: 0 +rank7 [2026-10-29 14:49:53,342] INFO : output_mode=set +rank7 [2026-10-29 14:49:53,343] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-29 14:49:53,343] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_pid parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_pt parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_eta parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : nn_energy parameters: 0 +rank7 [2026-10-29 14:49:53,343] INFO : Total MLPF parameters: 7260244 +rank7 [2026-10-29 14:49:53,344] INFO : MLPF __init__ done +rank7 [2026-10-29 14:49:53,345] INFO : Instantiated model +rank7 [2026-10-29 14:49:53,345] INFO : Moving model to device rank=7 +rank7 [2026-10-29 14:49:53,362] INFO : Moved model to device rank=7 +rank7 [2026-10-29 14:49:53,364] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-29 14:49:53,694] INFO : Restoring training from step 35001 +rank7 [2026-10-29 14:49:53,700] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/checkpoints/checkpoint-35000.pth +rank7 [2026-10-29 14:49:53,701] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-29 14:49:53,912] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-29 14:49:54,029] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank7 [2026-10-29 14:49:54,029] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-29 14:49:54,030] INFO : Trainable parameters: 7260244 +rank7 [2026-10-29 14:49:54,030] INFO : Non-trainable parameters: 0 +rank7 [2026-10-29 14:49:54,030] INFO : Total parameters: 7260244 +rank7 [2026-10-29 14:49:54,033] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-29 14:49:54,033] INFO : Getting dataloaders +rank7 [2026-10-29 14:49:54,033] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:01,421] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:02,340] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:03,556] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-29 14:50:03,557] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-29 14:50:03,557] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:03,820] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:04,076] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:04,390] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-29 14:50:04,390] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-29 14:50:18,172] INFO : Got dataloaders +rank7 [2026-10-29 14:50:18,172] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13892] of epochs [1] +rank7 [2026-10-29 14:50:18,173] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank7 [2026-10-29 14:50:18,173] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank7 [2026-10-29 14:50:18,173] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank7 [2026-10-29 14:50:19,543] INFO : loader split=train rank=7 len=21108 +rank7 [2026-10-29 14:50:19,543] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-29 14:50:19,543] INFO : Starting training from step 35001 to 50000 on rank 7 of 8 +rank7 [2026-10-29 14:50:19,543] INFO : Creating train_iterator +rank7 [2026-10-29 14:50:19,543] INFO : ResumableSampler physical:train loading state: start_index=222272 +rank7 [2026-10-29 14:50:33,911] INFO : Created train_iterator +rank7 [2026-10-29 14:54:16,364] INFO : Step 35100/50000 rank7 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6199 | Train Loss Components: Auxiliary: 0.2646 | Cardinality: 0.1291 | Classification: 0.1041 | Classification_binary: 0.1724 | HitGrouping: 0.2450 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1698 | Regression_eta: 0.0019 | Regression_pt: 0.1697 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0836 | Regression_eta: 0.0589 | Regression_pt: 0.0865 | Regression_sin_phi: 0.1209 | LR: 2.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 2.8691s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.5262 | max_loss: 0.1514 | max_backward: 1.0063 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.4499 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0054 | Memory: rss_MB: 4622.2 | uss_MB: 3387.4 | child_rss_MB: 5589.3 | allocated_MB: 371.3 | reserved_MB: 23786.0 | inactive_split_MB: 239.4 +rank7 [2026-10-29 14:57:53,685] INFO : Step 35200/50000 rank7 | Train Loss: 1.2667 | Train Loss Unweighted: 0.6288 | Train Loss Components: Auxiliary: 0.2791 | Cardinality: 0.1423 | Classification: 0.0996 | Classification_binary: 0.2076 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1584 | Regression_eta: 0.0017 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0799 | Regression_eta: 0.0694 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0617 | LR: 2.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5096s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1477 | max_backward: 1.5158 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.4384 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 4619.9 | uss_MB: 3390.3 | child_rss_MB: 5594.6 | allocated_MB: 371.7 | reserved_MB: 24792.0 | inactive_split_MB: 195.8 +rank7 [2026-10-29 15:01:33,154] INFO : Step 35300/50000 rank7 | Train Loss: 1.2683 | Train Loss Unweighted: 0.6653 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1401 | Classification: 0.0940 | Classification_binary: 0.1968 | HitGrouping: 0.2291 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1853 | Regression_eta: 0.0016 | Regression_pt: 0.1860 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0903 | Regression_eta: 0.0689 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0586 | LR: 2.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5922s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7170 | max_loss: 0.1684 | max_backward: 1.5168 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.5608 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4602.2 | uss_MB: 3365.4 | child_rss_MB: 5560.4 | allocated_MB: 369.9 | reserved_MB: 24792.0 | inactive_split_MB: 182.4 +rank7 [2026-10-29 15:05:10,670] INFO : Step 35400/50000 rank7 | Train Loss: 1.1769 | Train Loss Unweighted: 0.5989 | Train Loss Components: Auxiliary: 0.2499 | Cardinality: 0.1170 | Classification: 0.0862 | Classification_binary: 0.1554 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1764 | Regression_eta: 0.0018 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0852 | Regression_eta: 0.0462 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0499 | LR: 2.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.4264s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.6482 | max_loss: 0.2451 | max_backward: 1.3580 | max_optimizer_step: 0.2217 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4636.0 | uss_MB: 3403.5 | child_rss_MB: 5554.4 | allocated_MB: 369.1 | reserved_MB: 24792.0 | inactive_split_MB: 238.1 +rank7 [2026-10-29 15:08:49,316] INFO : Step 35500/50000 rank7 | Train Loss: 1.2600 | Train Loss Unweighted: 0.6285 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1586 | Classification: 0.0972 | Classification_binary: 0.1915 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0896 | Regression_eta: 0.0529 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0535 | LR: 2.74e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.4549s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.6686 | max_loss: 0.1553 | max_backward: 1.4007 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.5772 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4638.4 | uss_MB: 3409.7 | child_rss_MB: 5582.9 | allocated_MB: 371.3 | reserved_MB: 24792.0 | inactive_split_MB: 243.6 +rank7 [2026-10-29 15:12:27,419] INFO : Step 35600/50000 rank7 | Train Loss: 1.3076 | Train Loss Unweighted: 0.6533 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1549 | Classification: 0.0925 | Classification_binary: 0.2158 | HitGrouping: 0.2257 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1706 | Regression_eta: 0.0017 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0780 | Regression_eta: 0.0649 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0641 | LR: 2.72e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.6895s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.9197 | max_loss: 0.2566 | max_backward: 1.5032 | max_optimizer_step: 0.1481 | max_tensorboard_logging: 0.5803 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4639.0 | uss_MB: 3408.8 | child_rss_MB: 5584.6 | allocated_MB: 373.2 | reserved_MB: 24928.0 | inactive_split_MB: 299.0 +rank7 [2026-10-29 15:16:07,386] INFO : Step 35700/50000 rank7 | Train Loss: 1.1969 | Train Loss Unweighted: 0.5969 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1401 | Classification: 0.0864 | Classification_binary: 0.1643 | HitGrouping: 0.2239 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1710 | Regression_eta: 0.0016 | Regression_pt: 0.1717 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0826 | Regression_eta: 0.0382 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0471 | LR: 2.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.3519s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.8353 | max_loss: 0.2429 | max_backward: 1.2612 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.4688 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4643.0 | uss_MB: 3409.9 | child_rss_MB: 5586.5 | allocated_MB: 370.4 | reserved_MB: 24928.0 | inactive_split_MB: 249.6 +rank7 [2026-10-29 15:19:47,616] INFO : Step 35800/50000 rank7 | Train Loss: 1.2089 | Train Loss Unweighted: 0.6110 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1071 | Classification: 0.1027 | Classification_binary: 0.1828 | HitGrouping: 0.2252 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1600 | Regression_eta: 0.0018 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0751 | Regression_eta: 0.0529 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0680 | LR: 2.68e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.4880s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0167 | max_forward: 0.7409 | max_loss: 0.2158 | max_backward: 1.3682 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.6327 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4647.5 | uss_MB: 3415.0 | child_rss_MB: 5580.6 | allocated_MB: 370.9 | reserved_MB: 24928.0 | inactive_split_MB: 236.7 +rank7 [2026-10-29 15:23:30,653] INFO : Step 35900/50000 rank7 | Train Loss: 1.3182 | Train Loss Unweighted: 0.6839 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1412 | Classification: 0.1057 | Classification_binary: 0.2142 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1790 | Regression_eta: 0.0018 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0804 | Regression_eta: 0.0632 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0682 | LR: 2.65e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8615s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0075 | max_forward: 0.7775 | max_loss: 0.2538 | max_backward: 1.7522 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.5596 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0055 | Memory: rss_MB: 4651.2 | uss_MB: 3417.0 | child_rss_MB: 5596.6 | allocated_MB: 373.0 | reserved_MB: 24928.0 | inactive_split_MB: 264.3 +rank7 [2026-10-29 15:27:19,182] INFO : Step 36000/50000 rank7 | Train Loss: 1.2206 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2608 | Cardinality: 0.1459 | Classification: 0.0976 | Classification_binary: 0.1896 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1573 | Regression_eta: 0.0017 | Regression_pt: 0.1585 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0700 | Regression_eta: 0.0561 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0650 | LR: 2.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8283s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.6481 | max_loss: 0.2443 | max_backward: 1.2837 | max_optimizer_step: 0.2422 | max_tensorboard_logging: 0.9042 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0025 | Memory: rss_MB: 4651.4 | uss_MB: 3417.4 | child_rss_MB: 5572.0 | allocated_MB: 371.5 | reserved_MB: 25814.0 | inactive_split_MB: 268.8 +rank7 [2026-10-29 15:27:20,114] INFO : Running validation on rank7 +rank7 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 15:27:20,115] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 15:28:08,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:28:08,877] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 15:28:36,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 15:28:37,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 15:28:38,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 15:28:39,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 15:28:49,324] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 15:28:49,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:28:50,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 15:29:06,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 15:29:07,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 15:29:08,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 15:29:09,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 15:29:19,200] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 15:29:19,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:29:20,197] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 15:29:35,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 15:29:36,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 15:29:37,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 15:29:37,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_36000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 15:29:47,692] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 15:33:56,548] INFO : Step 36100/50000 rank7 | Train Loss: 1.1803 | Train Loss Unweighted: 0.6128 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.0876 | Classification: 0.0855 | Classification_binary: 0.1644 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1790 | Regression_eta: 0.0017 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0849 | Regression_eta: 0.0652 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0597 | LR: 2.61e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9092s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6677 | max_loss: 0.1463 | max_backward: 1.3494 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0151 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6382.6 | uss_MB: 5166.9 | child_rss_MB: 5572.5 | allocated_MB: 372.6 | reserved_MB: 22008.0 | inactive_split_MB: 180.8 +rank7 [2026-10-29 15:37:36,022] INFO : Step 36200/50000 rank7 | Train Loss: 1.3840 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1437 | Classification: 0.1086 | Classification_binary: 0.2129 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0021 | Regression_pt: 0.1775 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0873 | Regression_eta: 0.0602 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0724 | LR: 2.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5689s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6810 | max_loss: 0.1538 | max_backward: 1.4202 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4809 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 6381.3 | uss_MB: 5168.4 | child_rss_MB: 5586.2 | allocated_MB: 371.8 | reserved_MB: 23694.0 | inactive_split_MB: 190.4 +rank7 [2026-10-29 15:41:20,333] INFO : Step 36300/50000 rank7 | Train Loss: 1.2124 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1353 | Classification: 0.0866 | Classification_binary: 0.2024 | HitGrouping: 0.2195 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1561 | Regression_eta: 0.0016 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0841 | Regression_eta: 0.0484 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0565 | LR: 2.57e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5109s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7172 | max_loss: 0.1586 | max_backward: 1.3665 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.5347 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5468.6 | uss_MB: 4257.8 | child_rss_MB: 5611.0 | allocated_MB: 370.7 | reserved_MB: 24704.0 | inactive_split_MB: 279.6 +rank7 [2026-10-29 15:45:12,809] INFO : Step 36400/50000 rank7 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6001 | Train Loss Components: Auxiliary: 0.2470 | Cardinality: 0.1143 | Classification: 0.0935 | Classification_binary: 0.1670 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1778 | Regression_cos_phi: 0.1065 | Regression_energy: 0.0754 | Regression_eta: 0.0959 | Regression_pt: 0.0787 | Regression_sin_phi: 0.1087 | LR: 2.55e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.2464s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.5842 | max_loss: 0.1486 | max_backward: 1.1691 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5427.4 | uss_MB: 4259.0 | child_rss_MB: 5602.2 | allocated_MB: 368.8 | reserved_MB: 24704.0 | inactive_split_MB: 273.7 +rank7 [2026-10-29 15:49:02,009] INFO : Step 36500/50000 rank7 | Train Loss: 1.2408 | Train Loss Unweighted: 0.6217 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1359 | Classification: 0.1037 | Classification_binary: 0.1900 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1613 | Regression_eta: 0.0017 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0698 | Regression_eta: 0.0513 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0531 | LR: 2.52e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.0151s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 1.2401 | max_loss: 0.2278 | max_backward: 1.7898 | max_optimizer_step: 0.1594 | max_tensorboard_logging: 0.5259 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5425.1 | uss_MB: 4255.3 | child_rss_MB: 5604.3 | allocated_MB: 374.2 | reserved_MB: 24704.0 | inactive_split_MB: 201.0 +rank7 [2026-10-29 15:52:51,923] INFO : Step 36600/50000 rank7 | Train Loss: 1.3004 | Train Loss Unweighted: 0.6587 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1329 | Classification: 0.1049 | Classification_binary: 0.2093 | HitGrouping: 0.2265 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1692 | Regression_eta: 0.0018 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0751 | Regression_eta: 0.0657 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0870 | LR: 2.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7047s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0079 | max_forward: 0.7447 | max_loss: 0.2379 | max_backward: 1.4909 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5431.3 | uss_MB: 4264.8 | child_rss_MB: 5623.9 | allocated_MB: 373.4 | reserved_MB: 25722.0 | inactive_split_MB: 280.0 +rank7 [2026-10-29 15:56:51,302] INFO : Step 36700/50000 rank7 | Train Loss: 1.1796 | Train Loss Unweighted: 0.6007 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1046 | Classification: 0.0910 | Classification_binary: 0.1663 | HitGrouping: 0.2266 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1694 | Regression_eta: 0.0018 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0789 | Regression_eta: 0.0683 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0494 | LR: 2.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7866s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7241 | max_loss: 0.4062 | max_backward: 1.6662 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.5631 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0128 | Memory: rss_MB: 5429.8 | uss_MB: 4264.9 | child_rss_MB: 5608.9 | allocated_MB: 370.0 | reserved_MB: 25722.0 | inactive_split_MB: 235.8 +rank7 [2026-10-29 16:00:43,524] INFO : Step 36800/50000 rank7 | Train Loss: 1.3005 | Train Loss Unweighted: 0.6520 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1457 | Classification: 0.0947 | Classification_binary: 0.1878 | HitGrouping: 0.2368 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1823 | Regression_eta: 0.0018 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0762 | Regression_eta: 0.0550 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0634 | LR: 2.46e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4173s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6648 | max_loss: 0.1509 | max_backward: 1.4562 | max_optimizer_step: 0.1617 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5432.0 | uss_MB: 4268.7 | child_rss_MB: 5608.3 | allocated_MB: 374.1 | reserved_MB: 25722.0 | inactive_split_MB: 264.6 +rank7 [2026-10-29 16:04:39,810] INFO : Step 36900/50000 rank7 | Train Loss: 1.3468 | Train Loss Unweighted: 0.6929 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1423 | Classification: 0.0966 | Classification_binary: 0.2263 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0018 | Regression_pt: 0.1840 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2383 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0827 | Regression_eta: 0.0674 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1045 | LR: 2.44e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7922s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8608 | max_loss: 0.1500 | max_backward: 1.6460 | max_optimizer_step: 0.1591 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5432.9 | uss_MB: 4270.7 | child_rss_MB: 5612.1 | allocated_MB: 372.7 | reserved_MB: 25722.0 | inactive_split_MB: 201.9 +rank7 [2026-10-29 16:08:30,145] INFO : Step 37000/50000 rank7 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2667 | Cardinality: 0.1319 | Classification: 0.0832 | Classification_binary: 0.2004 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1739 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0870 | Regression_eta: 0.0486 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0533 | LR: 2.42e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9065s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0787 | max_forward: 0.7368 | max_loss: 0.4053 | max_backward: 1.5093 | max_optimizer_step: 0.2633 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0124 | Memory: rss_MB: 5433.3 | uss_MB: 4271.7 | child_rss_MB: 5612.3 | allocated_MB: 369.6 | reserved_MB: 25722.0 | inactive_split_MB: 245.5 +rank7 [2026-10-29 16:08:31,022] INFO : Running validation on rank7 +rank7 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:08:31,023] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:09:20,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:09:20,761] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 16:09:46,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:09:47,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:09:48,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:09:49,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:09:59,044] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:09:59,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:10:00,305] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 16:10:16,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:10:17,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:10:18,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:10:19,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:10:29,120] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 16:10:29,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:10:30,425] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 16:10:46,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:10:46,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:10:47,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:10:47,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_37000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:10:57,913] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 16:15:06,402] INFO : Step 37100/50000 rank7 | Train Loss: 1.1858 | Train Loss Unweighted: 0.5842 | Train Loss Components: Auxiliary: 0.2454 | Cardinality: 0.1320 | Classification: 0.0998 | Classification_binary: 0.1608 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0015 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0685 | Regression_eta: 0.0337 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0789 | LR: 2.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8495s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.5966 | max_loss: 0.4231 | max_backward: 1.2511 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5585 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0151 | Memory: rss_MB: 6352.3 | uss_MB: 5185.5 | child_rss_MB: 5643.1 | allocated_MB: 370.5 | reserved_MB: 23866.0 | inactive_split_MB: 230.7 +rank7 [2026-10-29 16:18:46,282] INFO : Step 37200/50000 rank7 | Train Loss: 1.2509 | Train Loss Unweighted: 0.6251 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1419 | Classification: 0.1001 | Classification_binary: 0.1897 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0016 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0872 | Regression_eta: 0.0390 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0492 | LR: 2.38e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2292s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.6965 | max_loss: 0.4467 | max_backward: 1.5915 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.9588 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 6352.2 | uss_MB: 5188.2 | child_rss_MB: 5619.1 | allocated_MB: 372.8 | reserved_MB: 24850.0 | inactive_split_MB: 202.6 +rank7 [2026-10-29 16:22:16,729] INFO : Step 37300/50000 rank7 | Train Loss: 1.3195 | Train Loss Unweighted: 0.6460 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1656 | Classification: 0.0891 | Classification_binary: 0.2051 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1731 | Regression_eta: 0.0018 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0772 | Regression_eta: 0.0481 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0564 | LR: 2.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5805s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0078 | max_forward: 0.7414 | max_loss: 0.4139 | max_backward: 1.7131 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.1591 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5540.0 | uss_MB: 4378.9 | child_rss_MB: 5638.9 | allocated_MB: 371.4 | reserved_MB: 24850.0 | inactive_split_MB: 472.2 +rank7 [2026-10-29 16:26:01,823] INFO : Step 37400/50000 rank7 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5722 | Train Loss Components: Auxiliary: 0.2494 | Cardinality: 0.1160 | Classification: 0.0726 | Classification_binary: 0.1621 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1673 | Regression_eta: 0.0015 | Regression_pt: 0.1671 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0705 | Regression_eta: 0.0674 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0741 | LR: 2.34e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9404s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0070 | max_forward: 0.6234 | max_loss: 0.4104 | max_backward: 1.4021 | max_optimizer_step: 0.1717 | max_tensorboard_logging: 0.9360 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5545.8 | uss_MB: 4382.5 | child_rss_MB: 5616.2 | allocated_MB: 372.1 | reserved_MB: 25864.0 | inactive_split_MB: 252.3 +rank7 [2026-10-29 16:30:27,637] INFO : Step 37500/50000 rank7 | Train Loss: 1.2622 | Train Loss Unweighted: 0.6417 | Train Loss Components: Auxiliary: 0.2735 | Cardinality: 0.1337 | Classification: 0.1053 | Classification_binary: 0.1880 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1723 | Regression_eta: 0.0018 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0855 | Regression_eta: 0.0781 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0798 | LR: 2.32e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3479s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7284 | max_loss: 0.2535 | max_backward: 1.7442 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.9221 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5549.1 | uss_MB: 4387.0 | child_rss_MB: 5615.9 | allocated_MB: 373.1 | reserved_MB: 25864.0 | inactive_split_MB: 241.6 +rank7 [2026-10-29 16:34:14,939] INFO : Step 37600/50000 rank7 | Train Loss: 1.3614 | Train Loss Unweighted: 0.6993 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.1290 | Classification: 0.1043 | Classification_binary: 0.2211 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1844 | Regression_eta: 0.0019 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0789 | Regression_eta: 0.0680 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0993 | LR: 2.30e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.1114s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.9437 | max_loss: 0.4035 | max_backward: 1.5693 | max_optimizer_step: 0.2687 | max_tensorboard_logging: 0.6215 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5547.5 | uss_MB: 4389.3 | child_rss_MB: 5618.4 | allocated_MB: 373.5 | reserved_MB: 25864.0 | inactive_split_MB: 248.5 +rank7 [2026-10-29 16:38:00,808] INFO : Step 37700/50000 rank7 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6262 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1506 | Classification: 0.0864 | Classification_binary: 0.1791 | HitGrouping: 0.2367 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1775 | Regression_eta: 0.0023 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0954 | Regression_eta: 0.0767 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0756 | LR: 2.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1664s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6796 | max_loss: 0.2568 | max_backward: 1.6004 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9011 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5543.3 | uss_MB: 4389.0 | child_rss_MB: 5639.6 | allocated_MB: 370.4 | reserved_MB: 25864.0 | inactive_split_MB: 212.4 +rank7 [2026-10-29 16:41:51,409] INFO : Step 37800/50000 rank7 | Train Loss: 1.2482 | Train Loss Unweighted: 0.6211 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1395 | Classification: 0.0981 | Classification_binary: 0.1758 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1713 | Regression_eta: 0.0017 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1562 | Regression_cos_phi: 0.1167 | Regression_energy: 0.0670 | Regression_eta: 0.0724 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0918 | LR: 2.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.2916s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.6313 | max_loss: 0.1452 | max_backward: 1.2659 | max_optimizer_step: 0.1683 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5533.4 | uss_MB: 4392.6 | child_rss_MB: 5634.3 | allocated_MB: 372.8 | reserved_MB: 25864.0 | inactive_split_MB: 248.0 +rank7 [2026-10-29 16:45:39,811] INFO : Step 37900/50000 rank7 | Train Loss: 1.3053 | Train Loss Unweighted: 0.6403 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1623 | Classification: 0.0991 | Classification_binary: 0.2167 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1598 | Regression_eta: 0.0016 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0714 | Regression_eta: 0.0568 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0602 | LR: 2.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8070s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.6938 | max_loss: 0.4358 | max_backward: 1.6245 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 0.5273 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5532.7 | uss_MB: 4393.0 | child_rss_MB: 5623.0 | allocated_MB: 371.9 | reserved_MB: 25864.0 | inactive_split_MB: 269.1 +rank7 [2026-10-29 16:49:32,191] INFO : Step 38000/50000 rank7 | Train Loss: 1.2294 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2563 | Cardinality: 0.1241 | Classification: 0.0905 | Classification_binary: 0.1892 | HitGrouping: 0.2219 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1597 | Regression_eta: 0.0022 | Regression_pt: 0.1608 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0677 | Regression_eta: 0.0665 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0807 | LR: 2.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8885s | Phase Times: max_pre_step_barrier: 0.1093 | max_batch_to_device: 0.0067 | max_forward: 0.7038 | max_loss: 0.4218 | max_backward: 1.4728 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.5938 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0194 | Memory: rss_MB: 5525.9 | uss_MB: 4394.8 | child_rss_MB: 5621.6 | allocated_MB: 372.5 | reserved_MB: 25864.0 | inactive_split_MB: 195.5 +rank7 [2026-10-29 16:49:33,136] INFO : Running validation on rank7 +rank7 [2026-10-29 16:49:33,137] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:49:33,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:50:20,730] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:50:21,276] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 16:50:47,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:50:48,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:50:51,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:50:53,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:51:03,484] INFO : Time taken to make predictions on device 7 is: 0.29 min +rank7 [2026-10-29 16:51:06,261] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:51:06,743] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 16:51:22,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:51:23,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:51:24,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:51:25,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:51:35,659] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:51:36,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:51:36,911] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 16:51:52,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:51:52,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:51:53,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:51:54,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_38000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:52:04,224] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 16:56:13,530] INFO : Step 38100/50000 rank7 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5831 | Train Loss Components: Auxiliary: 0.2474 | Cardinality: 0.1369 | Classification: 0.0835 | Classification_binary: 0.1630 | HitGrouping: 0.2277 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1661 | Regression_eta: 0.0014 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0794 | Regression_eta: 0.0482 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0698 | LR: 2.20e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3706s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8554 | max_loss: 0.2103 | max_backward: 1.1447 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 1.3161 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5660.5 | uss_MB: 4526.5 | child_rss_MB: 5618.6 | allocated_MB: 372.9 | reserved_MB: 23120.0 | inactive_split_MB: 242.2 +rank7 [2026-10-29 17:00:01,012] INFO : Step 38200/50000 rank7 | Train Loss: 1.3148 | Train Loss Unweighted: 0.6719 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1286 | Classification: 0.1159 | Classification_binary: 0.2032 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0019 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0755 | Regression_eta: 0.0675 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0738 | LR: 2.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8293s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8438 | max_loss: 0.4169 | max_backward: 1.6983 | max_optimizer_step: 0.1741 | max_tensorboard_logging: 0.4797 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 5664.3 | uss_MB: 3857.7 | child_rss_MB: 5625.0 | allocated_MB: 372.2 | reserved_MB: 24114.0 | inactive_split_MB: 223.8 +rank7 [2026-10-29 17:03:44,995] INFO : Step 38300/50000 rank7 | Train Loss: 1.3280 | Train Loss Unweighted: 0.6582 | Train Loss Components: Auxiliary: 0.2885 | Cardinality: 0.1578 | Classification: 0.0933 | Classification_binary: 0.2126 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1740 | Regression_eta: 0.0017 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.2019 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0845 | Regression_eta: 0.0560 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0980 | LR: 2.16e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.2348s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.9351 | max_loss: 0.4040 | max_backward: 1.9466 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.5666 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5665.9 | uss_MB: 4532.3 | child_rss_MB: 5604.0 | allocated_MB: 374.3 | reserved_MB: 24296.0 | inactive_split_MB: 239.7 +rank7 [2026-10-29 17:07:28,967] INFO : Step 38400/50000 rank7 | Train Loss: 1.1844 | Train Loss Unweighted: 0.5977 | Train Loss Components: Auxiliary: 0.2522 | Cardinality: 0.1419 | Classification: 0.0892 | Classification_binary: 0.1661 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1688 | Regression_eta: 0.0014 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0715 | Regression_eta: 0.0509 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0701 | LR: 2.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5972s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7066 | max_loss: 0.4140 | max_backward: 1.3562 | max_optimizer_step: 0.1730 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5663.6 | uss_MB: 4531.2 | child_rss_MB: 5642.0 | allocated_MB: 369.3 | reserved_MB: 25138.0 | inactive_split_MB: 295.6 +rank7 [2026-10-29 17:11:17,145] INFO : Step 38500/50000 rank7 | Train Loss: 1.1674 | Train Loss Unweighted: 0.5805 | Train Loss Components: Auxiliary: 0.2538 | Cardinality: 0.1427 | Classification: 0.0846 | Classification_binary: 0.1704 | HitGrouping: 0.2177 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1605 | Regression_eta: 0.0014 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0797 | Regression_eta: 0.0355 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0473 | LR: 2.12e-04 | DataLoad Time: 0.0225s | Model Forward Time: 3.5362s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.6291 | max_loss: 0.1671 | max_backward: 1.3787 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.5507 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5664.3 | uss_MB: 4534.1 | child_rss_MB: 5654.2 | allocated_MB: 371.4 | reserved_MB: 25138.0 | inactive_split_MB: 298.3 +rank7 [2026-10-29 17:15:06,552] INFO : Step 38600/50000 rank7 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6294 | Train Loss Components: Auxiliary: 0.2766 | Cardinality: 0.1443 | Classification: 0.0945 | Classification_binary: 0.2155 | HitGrouping: 0.2230 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1572 | Regression_eta: 0.0016 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1818 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0753 | Regression_eta: 0.0636 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0758 | LR: 2.11e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4121s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7338 | max_loss: 0.1603 | max_backward: 1.5854 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.1254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5662.5 | uss_MB: 4536.7 | child_rss_MB: 5616.9 | allocated_MB: 371.3 | reserved_MB: 25796.0 | inactive_split_MB: 267.6 +rank7 [2026-10-29 17:18:55,030] INFO : Step 38700/50000 rank7 | Train Loss: 1.1924 | Train Loss Unweighted: 0.5742 | Train Loss Components: Auxiliary: 0.2569 | Cardinality: 0.1575 | Classification: 0.0753 | Classification_binary: 0.1799 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1577 | Regression_eta: 0.0015 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0787 | Regression_eta: 0.0492 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0517 | LR: 2.09e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8735s | Phase Times: max_pre_step_barrier: 0.0090 | max_batch_to_device: 0.0071 | max_forward: 0.7002 | max_loss: 0.1771 | max_backward: 1.5033 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 0.6458 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5655.8 | uss_MB: 4536.6 | child_rss_MB: 5616.9 | allocated_MB: 371.9 | reserved_MB: 25796.0 | inactive_split_MB: 298.6 +rank7 [2026-10-29 17:22:44,406] INFO : Step 38800/50000 rank7 | Train Loss: 1.1386 | Train Loss Unweighted: 0.5780 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1160 | Classification: 0.0872 | Classification_binary: 0.1604 | HitGrouping: 0.2267 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1625 | Regression_eta: 0.0018 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0823 | Regression_eta: 0.1201 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0498 | LR: 2.07e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5801s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1010 | max_forward: 0.6780 | max_loss: 0.1419 | max_backward: 1.4809 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5657.5 | uss_MB: 4539.9 | child_rss_MB: 5623.1 | allocated_MB: 368.9 | reserved_MB: 25796.0 | inactive_split_MB: 297.6 +rank7 [2026-10-29 17:26:35,864] INFO : Step 38900/50000 rank7 | Train Loss: 1.2551 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2750 | Cardinality: 0.1441 | Classification: 0.1007 | Classification_binary: 0.2005 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1581 | Regression_eta: 0.0017 | Regression_pt: 0.1588 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0833 | Regression_eta: 0.0510 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0640 | LR: 2.05e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6668s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.0434 | max_loss: 0.4747 | max_backward: 2.3052 | max_optimizer_step: 0.1764 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0806 | Memory: rss_MB: 5657.8 | uss_MB: 4541.0 | child_rss_MB: 5620.6 | allocated_MB: 372.7 | reserved_MB: 25796.0 | inactive_split_MB: 297.4 +rank7 [2026-10-29 17:30:46,441] INFO : Step 39000/50000 rank7 | Train Loss: 1.2081 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2609 | Cardinality: 0.1532 | Classification: 0.0853 | Classification_binary: 0.1861 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0014 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0716 | Regression_eta: 0.0384 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0508 | LR: 2.03e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.6464s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7943 | max_loss: 0.1493 | max_backward: 1.3186 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5214 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5658.3 | uss_MB: 4542.5 | child_rss_MB: 5646.9 | allocated_MB: 371.2 | reserved_MB: 25796.0 | inactive_split_MB: 285.1 +rank7 [2026-10-29 17:30:47,444] INFO : Running validation on rank7 +rank7 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 17:30:47,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 17:31:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:31:36,204] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 17:32:01,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 17:32:02,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 17:32:03,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 17:32:04,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 17:32:14,644] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 17:32:15,357] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:32:16,020] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 17:32:31,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 17:32:32,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 17:32:33,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 17:32:34,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 17:32:44,368] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 17:32:45,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:32:45,862] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 17:33:01,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 17:33:01,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 17:33:02,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 17:33:02,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_39000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 17:33:12,992] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 17:37:22,844] INFO : Step 39100/50000 rank7 | Train Loss: 1.1820 | Train Loss Unweighted: 0.6017 | Train Loss Components: Auxiliary: 0.2482 | Cardinality: 0.1244 | Classification: 0.0931 | Classification_binary: 0.1665 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1689 | Regression_eta: 0.0016 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1893 | Regression_cos_phi: 0.1057 | Regression_energy: 0.0904 | Regression_eta: 0.0636 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0706 | LR: 2.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.6370s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0059 | max_forward: 0.6844 | max_loss: 0.4092 | max_backward: 1.3095 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.5751 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 7392.3 | uss_MB: 6269.2 | child_rss_MB: 5633.2 | allocated_MB: 370.2 | reserved_MB: 23274.0 | inactive_split_MB: 245.0 +rank7 [2026-10-29 17:41:18,055] INFO : Step 39200/50000 rank7 | Train Loss: 1.2555 | Train Loss Unweighted: 0.6299 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1360 | Classification: 0.1015 | Classification_binary: 0.1910 | HitGrouping: 0.2236 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1662 | Regression_eta: 0.0018 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0776 | Regression_eta: 0.0615 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0769 | LR: 2.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7436s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0223 | max_forward: 0.6943 | max_loss: 0.4517 | max_backward: 1.5485 | max_optimizer_step: 0.1758 | max_tensorboard_logging: 0.5287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 5736.0 | uss_MB: 4613.5 | child_rss_MB: 5658.1 | allocated_MB: 371.1 | reserved_MB: 24136.0 | inactive_split_MB: 217.0 +rank7 [2026-10-29 17:45:05,746] INFO : Step 39300/50000 rank7 | Train Loss: 1.3336 | Train Loss Unweighted: 0.6529 | Train Loss Components: Auxiliary: 0.2875 | Cardinality: 0.1542 | Classification: 0.0964 | Classification_binary: 0.2178 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1670 | Regression_eta: 0.0019 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0891 | Classification_binary: 0.2354 | Regression_cos_phi: 0.1092 | Regression_energy: 0.0764 | Regression_eta: 0.0899 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0960 | LR: 1.98e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9346s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7567 | max_loss: 0.1516 | max_backward: 1.6025 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.5723 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5734.3 | uss_MB: 4614.5 | child_rss_MB: 5658.8 | allocated_MB: 372.6 | reserved_MB: 25040.0 | inactive_split_MB: 264.5 +rank7 [2026-10-29 17:49:01,211] INFO : Step 39400/50000 rank7 | Train Loss: 1.1531 | Train Loss Unweighted: 0.5747 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1090 | Classification: 0.0824 | Classification_binary: 0.1684 | HitGrouping: 0.2247 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1601 | Regression_eta: 0.0017 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0846 | Regression_eta: 0.0891 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0932 | LR: 1.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0456s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6628 | max_loss: 0.2280 | max_backward: 1.5740 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.7909 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1167 | Memory: rss_MB: 5735.9 | uss_MB: 4616.4 | child_rss_MB: 5633.3 | allocated_MB: 371.0 | reserved_MB: 25042.0 | inactive_split_MB: 243.5 +rank7 [2026-10-29 17:53:06,263] INFO : Step 39500/50000 rank7 | Train Loss: 1.1827 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2582 | Cardinality: 0.1306 | Classification: 0.0856 | Classification_binary: 0.1712 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1694 | Regression_eta: 0.0015 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0895 | Regression_eta: 0.0506 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0769 | LR: 1.94e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.2572s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9787 | max_loss: 0.1526 | max_backward: 1.5818 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.9606 | max_tensorboard_flush: 0.0143 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5734.9 | uss_MB: 4617.8 | child_rss_MB: 5655.0 | allocated_MB: 372.3 | reserved_MB: 25042.0 | inactive_split_MB: 205.7 +rank7 [2026-10-29 17:56:59,043] INFO : Step 39600/50000 rank7 | Train Loss: 1.2503 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2770 | Cardinality: 0.1423 | Classification: 0.0942 | Classification_binary: 0.1962 | HitGrouping: 0.2259 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1682 | Regression_eta: 0.0016 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0741 | Regression_eta: 0.0401 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0544 | LR: 1.93e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1943s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7285 | max_loss: 0.1695 | max_backward: 1.4499 | max_optimizer_step: 0.2733 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5741.8 | uss_MB: 4609.5 | child_rss_MB: 5647.8 | allocated_MB: 372.1 | reserved_MB: 25042.0 | inactive_split_MB: 240.7 +rank7 [2026-10-29 18:00:43,549] INFO : Step 39700/50000 rank7 | Train Loss: 1.2473 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1412 | Classification: 0.0893 | Classification_binary: 0.1979 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0817 | Regression_eta: 0.0597 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0745 | LR: 1.91e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2351s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.8353 | max_loss: 0.1499 | max_backward: 1.5533 | max_optimizer_step: 0.1751 | max_tensorboard_logging: 0.9194 | max_tensorboard_flush: 0.0194 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5739.5 | uss_MB: 4622.2 | child_rss_MB: 5655.8 | allocated_MB: 371.4 | reserved_MB: 26076.0 | inactive_split_MB: 175.0 +rank7 [2026-10-29 18:04:32,041] INFO : Step 39800/50000 rank7 | Train Loss: 1.1257 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2410 | Cardinality: 0.1288 | Classification: 0.0744 | Classification_binary: 0.1604 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1564 | Regression_eta: 0.0014 | Regression_pt: 0.1564 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0339 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0760 | Regression_eta: 0.0462 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0634 | LR: 1.89e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8992s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8615 | max_loss: 0.2440 | max_backward: 1.3907 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.8493 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 5735.4 | uss_MB: 4623.3 | child_rss_MB: 5655.5 | allocated_MB: 369.2 | reserved_MB: 26076.0 | inactive_split_MB: 216.0 +rank7 [2026-10-29 18:08:19,424] INFO : Step 39900/50000 rank7 | Train Loss: 1.2938 | Train Loss Unweighted: 0.6422 | Train Loss Components: Auxiliary: 0.2825 | Cardinality: 0.1479 | Classification: 0.1002 | Classification_binary: 0.1983 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1696 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0568 | Regression_energy: 0.1030 | Regression_eta: 0.0543 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0675 | LR: 1.88e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.9488 | max_loss: 0.2481 | max_backward: 1.5690 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.8768 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 5734.0 | uss_MB: 4624.7 | child_rss_MB: 5656.8 | allocated_MB: 372.2 | reserved_MB: 26076.0 | inactive_split_MB: 186.7 +rank7 [2026-10-29 18:12:05,434] INFO : Step 40000/50000 rank7 | Train Loss: 1.2693 | Train Loss Unweighted: 0.6322 | Train Loss Components: Auxiliary: 0.2754 | Cardinality: 0.1360 | Classification: 0.0842 | Classification_binary: 0.2093 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0017 | Regression_pt: 0.1675 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.2215 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0859 | Regression_eta: 0.0653 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0718 | LR: 1.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.9410s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7405 | max_loss: 0.1967 | max_backward: 1.6147 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5735.3 | uss_MB: 4629.2 | child_rss_MB: 5657.6 | allocated_MB: 372.7 | reserved_MB: 26076.0 | inactive_split_MB: 258.7 +rank7 [2026-10-29 18:12:06,328] INFO : Running validation on rank7 +rank7 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:12:06,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:12:59,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:12:59,648] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 18:13:23,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:13:24,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:13:25,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:13:26,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:13:36,336] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 18:13:37,231] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:13:37,720] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 18:13:53,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:13:54,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:13:55,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:13:56,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:14:06,374] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:14:06,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:14:07,479] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 18:14:22,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:14:23,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:14:24,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:14:24,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_40000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:14:34,860] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 18:18:48,529] INFO : Step 40100/50000 rank7 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5878 | Train Loss Components: Auxiliary: 0.2554 | Cardinality: 0.1206 | Classification: 0.0793 | Classification_binary: 0.1671 | HitGrouping: 0.2378 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1687 | Regression_eta: 0.0015 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0727 | Regression_eta: 0.0578 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0602 | LR: 1.84e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.1861s | Phase Times: max_pre_step_barrier: 0.0248 | max_batch_to_device: 0.0169 | max_forward: 1.0478 | max_loss: 0.2037 | max_backward: 1.9625 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 7388.0 | uss_MB: 6198.7 | child_rss_MB: 5632.6 | allocated_MB: 370.0 | reserved_MB: 24300.0 | inactive_split_MB: 278.5 +rank7 [2026-10-29 18:22:51,411] INFO : Step 40200/50000 rank7 | Train Loss: 1.2410 | Train Loss Unweighted: 0.6168 | Train Loss Components: Auxiliary: 0.2665 | Cardinality: 0.1384 | Classification: 0.0910 | Classification_binary: 0.1769 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1724 | Regression_eta: 0.0017 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0731 | Regression_eta: 0.0539 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0705 | LR: 1.83e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.9278s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.6645 | max_loss: 0.1444 | max_backward: 1.3753 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.9352 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5840.4 | uss_MB: 4731.9 | child_rss_MB: 5605.5 | allocated_MB: 371.5 | reserved_MB: 24462.0 | inactive_split_MB: 203.1 +rank7 [2026-10-29 18:26:53,014] INFO : Step 40300/50000 rank7 | Train Loss: 1.3862 | Train Loss Unweighted: 0.6941 | Train Loss Components: Auxiliary: 0.3058 | Cardinality: 0.1629 | Classification: 0.1025 | Classification_binary: 0.2232 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1819 | Regression_eta: 0.0018 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2282 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0955 | Regression_eta: 0.0622 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0623 | LR: 1.81e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4876s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.8633 | max_loss: 0.4528 | max_backward: 1.6054 | max_optimizer_step: 0.2773 | max_tensorboard_logging: 0.9251 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5838.9 | uss_MB: 4736.1 | child_rss_MB: 5590.9 | allocated_MB: 371.8 | reserved_MB: 24462.0 | inactive_split_MB: 275.5 +rank7 [2026-10-29 18:30:44,152] INFO : Step 40400/50000 rank7 | Train Loss: 1.2626 | Train Loss Unweighted: 0.6257 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1510 | Classification: 0.0953 | Classification_binary: 0.1822 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1712 | Regression_eta: 0.0017 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0902 | Regression_eta: 0.0462 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0498 | LR: 1.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0120s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7355 | max_loss: 0.4049 | max_backward: 1.3917 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.8018 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0720 | Memory: rss_MB: 5830.8 | uss_MB: 4733.4 | child_rss_MB: 5608.9 | allocated_MB: 371.1 | reserved_MB: 25306.0 | inactive_split_MB: 214.9 +rank7 [2026-10-29 18:34:31,872] INFO : Step 40500/50000 rank7 | Train Loss: 1.2418 | Train Loss Unweighted: 0.6387 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1362 | Classification: 0.0900 | Classification_binary: 0.1681 | HitGrouping: 0.2360 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1873 | Regression_eta: 0.0014 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0969 | Regression_eta: 0.0587 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0614 | LR: 1.78e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.4809s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0345 | max_forward: 0.6072 | max_loss: 0.4176 | max_backward: 1.3396 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4742 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5829.5 | uss_MB: 4736.5 | child_rss_MB: 5562.3 | allocated_MB: 370.5 | reserved_MB: 26314.0 | inactive_split_MB: 220.6 +rank7 [2026-10-29 18:38:19,542] INFO : Step 40600/50000 rank7 | Train Loss: 1.3511 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2971 | Cardinality: 0.1354 | Classification: 0.1057 | Classification_binary: 0.2249 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1896 | Regression_eta: 0.0017 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0909 | Regression_eta: 0.0580 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0908 | LR: 1.76e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1588s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0072 | max_forward: 0.8547 | max_loss: 0.3993 | max_backward: 1.5665 | max_optimizer_step: 0.2585 | max_tensorboard_logging: 0.6065 | max_tensorboard_flush: 0.0023 | max_post_step_barrier: 0.0078 | Memory: rss_MB: 5827.8 | uss_MB: 3919.0 | child_rss_MB: 5605.9 | allocated_MB: 371.4 | reserved_MB: 26314.0 | inactive_split_MB: 261.2 +rank7 [2026-10-29 18:42:09,670] INFO : Step 40700/50000 rank7 | Train Loss: 1.2581 | Train Loss Unweighted: 0.6368 | Train Loss Components: Auxiliary: 0.2753 | Cardinality: 0.1539 | Classification: 0.0907 | Classification_binary: 0.1937 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1751 | Regression_eta: 0.0015 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0791 | Regression_eta: 0.0681 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0513 | LR: 1.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5725s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0721 | max_forward: 0.7614 | max_loss: 0.1418 | max_backward: 1.3291 | max_optimizer_step: 0.1779 | max_tensorboard_logging: 0.4600 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5828.3 | uss_MB: 4737.7 | child_rss_MB: 5561.6 | allocated_MB: 370.6 | reserved_MB: 26314.0 | inactive_split_MB: 224.9 +rank7 [2026-10-29 18:45:59,792] INFO : Step 40800/50000 rank7 | Train Loss: 1.1626 | Train Loss Unweighted: 0.5950 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1286 | Classification: 0.0858 | Classification_binary: 0.1495 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1779 | Regression_eta: 0.0014 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0828 | Regression_eta: 0.0400 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0465 | LR: 1.73e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.0685s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0780 | max_forward: 0.6908 | max_loss: 0.3981 | max_backward: 1.6216 | max_optimizer_step: 0.2669 | max_tensorboard_logging: 0.6596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5832.9 | uss_MB: 4742.6 | child_rss_MB: 5567.6 | allocated_MB: 372.0 | reserved_MB: 26314.0 | inactive_split_MB: 278.5 +rank7 [2026-10-29 18:49:49,167] INFO : Step 40900/50000 rank7 | Train Loss: 1.2554 | Train Loss Unweighted: 0.6393 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1234 | Classification: 0.0974 | Classification_binary: 0.1946 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1709 | Regression_eta: 0.0016 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0820 | Regression_eta: 0.0619 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0781 | LR: 1.72e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6409s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6694 | max_loss: 0.2470 | max_backward: 1.3512 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.5795 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5780.2 | uss_MB: 4745.0 | child_rss_MB: 5393.0 | allocated_MB: 372.4 | reserved_MB: 26314.0 | inactive_split_MB: 181.6 +rank7 [2026-10-29 18:53:41,415] INFO : Step 41000/50000 rank7 | Train Loss: 1.2889 | Train Loss Unweighted: 0.6706 | Train Loss Components: Auxiliary: 0.2844 | Cardinality: 0.1395 | Classification: 0.0865 | Classification_binary: 0.2124 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1832 | Regression_eta: 0.0016 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2314 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0822 | Regression_eta: 0.0546 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0949 | LR: 1.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4686s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8532 | max_loss: 0.1499 | max_backward: 1.7339 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 0.5948 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5776.8 | uss_MB: 4526.0 | child_rss_MB: 5381.0 | allocated_MB: 368.6 | reserved_MB: 26314.0 | inactive_split_MB: 268.9 +rank7 [2026-10-29 18:53:42,384] INFO : Running validation on rank7 +rank7 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:53:42,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:54:33,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:54:33,853] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 18:54:59,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:55:00,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:55:01,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:55:02,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:55:12,516] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 18:55:13,077] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:55:13,607] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 18:55:29,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:55:30,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:55:31,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:55:32,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:55:42,615] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:55:43,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:55:43,851] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 18:55:59,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:55:59,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:56:00,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:56:01,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_41000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:56:11,056] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 19:00:33,893] INFO : Step 41100/50000 rank7 | Train Loss: 1.2011 | Train Loss Unweighted: 0.5953 | Train Loss Components: Auxiliary: 0.2613 | Cardinality: 0.1342 | Classification: 0.0765 | Classification_binary: 0.1752 | HitGrouping: 0.2282 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1698 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0873 | Regression_eta: 0.0704 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0670 | LR: 1.69e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8492s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7032 | max_loss: 0.4058 | max_backward: 1.5388 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.5577 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 7565.6 | uss_MB: 6530.2 | child_rss_MB: 5399.9 | allocated_MB: 369.4 | reserved_MB: 22646.0 | inactive_split_MB: 236.1 +rank7 [2026-10-29 19:04:28,228] INFO : Step 41200/50000 rank7 | Train Loss: 1.2785 | Train Loss Unweighted: 0.6915 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1139 | Classification: 0.0879 | Classification_binary: 0.1871 | HitGrouping: 0.2431 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2061 | Regression_eta: 0.0017 | Regression_pt: 0.2070 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0866 | Regression_energy: 0.1022 | Regression_eta: 0.0680 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0784 | LR: 1.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9454s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0076 | max_forward: 0.6641 | max_loss: 0.4191 | max_backward: 1.4821 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0502 | Memory: rss_MB: 5889.9 | uss_MB: 4855.2 | child_rss_MB: 5412.0 | allocated_MB: 373.5 | reserved_MB: 23818.0 | inactive_split_MB: 202.7 +rank7 [2026-10-29 19:08:18,072] INFO : Step 41300/50000 rank7 | Train Loss: 1.3323 | Train Loss Unweighted: 0.7063 | Train Loss Components: Auxiliary: 0.2932 | Cardinality: 0.1357 | Classification: 0.0987 | Classification_binary: 0.2174 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1926 | Regression_eta: 0.0017 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0845 | Regression_eta: 0.0448 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0544 | LR: 1.66e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1394s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1715 | max_backward: 1.6081 | max_optimizer_step: 0.2114 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 5896.4 | uss_MB: 4859.7 | child_rss_MB: 5400.6 | allocated_MB: 369.8 | reserved_MB: 24806.0 | inactive_split_MB: 204.9 +rank7 [2026-10-29 19:12:02,347] INFO : Step 41400/50000 rank7 | Train Loss: 1.2045 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1191 | Classification: 0.0893 | Classification_binary: 0.1853 | HitGrouping: 0.2301 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1705 | Regression_eta: 0.0015 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0806 | Regression_eta: 0.0541 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0773 | LR: 1.64e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9381s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7497 | max_loss: 0.2568 | max_backward: 1.6855 | max_optimizer_step: 0.1782 | max_tensorboard_logging: 0.5412 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5896.2 | uss_MB: 4860.7 | child_rss_MB: 5413.5 | allocated_MB: 369.6 | reserved_MB: 25822.0 | inactive_split_MB: 196.4 +rank7 [2026-10-29 19:15:48,707] INFO : Step 41500/50000 rank7 | Train Loss: 1.1757 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1153 | Classification: 0.1016 | Classification_binary: 0.1600 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0793 | Regression_eta: 0.0604 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0729 | LR: 1.63e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.3266s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.5640 | max_loss: 0.2232 | max_backward: 1.0877 | max_optimizer_step: 0.2641 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5893.9 | uss_MB: 4862.5 | child_rss_MB: 5417.0 | allocated_MB: 368.9 | reserved_MB: 25822.0 | inactive_split_MB: 180.7 +rank7 [2026-10-29 19:19:48,125] INFO : Step 41600/50000 rank7 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6769 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1622 | Classification: 0.1041 | Classification_binary: 0.2083 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1799 | Regression_eta: 0.0017 | Regression_pt: 0.1810 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0935 | Regression_eta: 0.0543 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0427 | LR: 1.61e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.7660s | Phase Times: max_pre_step_barrier: 0.1808 | max_batch_to_device: 0.0065 | max_forward: 0.6911 | max_loss: 0.1527 | max_backward: 1.3352 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.5149 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 5898.3 | uss_MB: 4863.9 | child_rss_MB: 5407.4 | allocated_MB: 373.9 | reserved_MB: 26836.0 | inactive_split_MB: 259.0 +rank7 [2026-10-29 19:23:41,843] INFO : Step 41700/50000 rank7 | Train Loss: 1.3039 | Train Loss Unweighted: 0.6667 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1539 | Classification: 0.0911 | Classification_binary: 0.2081 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1816 | Regression_eta: 0.0017 | Regression_pt: 0.1825 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0778 | Regression_eta: 0.0541 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0722 | LR: 1.60e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.9470s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7654 | max_loss: 0.2545 | max_backward: 1.7417 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.4664 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5902.0 | uss_MB: 4866.3 | child_rss_MB: 5386.3 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 294.7 +rank7 [2026-10-29 19:27:38,847] INFO : Step 41800/50000 rank7 | Train Loss: 1.2359 | Train Loss Unweighted: 0.6232 | Train Loss Components: Auxiliary: 0.2666 | Cardinality: 0.1256 | Classification: 0.0814 | Classification_binary: 0.1721 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1832 | Regression_eta: 0.0019 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0573 | Regression_energy: 0.1011 | Regression_eta: 0.0512 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0660 | LR: 1.58e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.3160s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6778 | max_loss: 0.4101 | max_backward: 2.9418 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.6363 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5905.9 | uss_MB: 4867.4 | child_rss_MB: 5424.7 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 289.8 +rank7 [2026-10-29 19:31:32,457] INFO : Step 41900/50000 rank7 | Train Loss: 1.2620 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2773 | Cardinality: 0.1168 | Classification: 0.0914 | Classification_binary: 0.1821 | HitGrouping: 0.2350 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1855 | Regression_eta: 0.0016 | Regression_pt: 0.1867 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0970 | Regression_eta: 0.0509 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0789 | LR: 1.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4514s | Phase Times: max_pre_step_barrier: 0.1109 | max_batch_to_device: 0.0073 | max_forward: 0.6155 | max_loss: 0.1410 | max_backward: 1.2192 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5902.3 | uss_MB: 4705.1 | child_rss_MB: 5380.8 | allocated_MB: 370.3 | reserved_MB: 26836.0 | inactive_split_MB: 175.7 +rank7 [2026-10-29 19:35:32,890] INFO : Step 42000/50000 rank7 | Train Loss: 1.4172 | Train Loss Unweighted: 0.7107 | Train Loss Components: Auxiliary: 0.3116 | Cardinality: 0.1538 | Classification: 0.1003 | Classification_binary: 0.2292 | HitGrouping: 0.2457 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0018 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.2361 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0855 | Regression_eta: 0.0622 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0817 | LR: 1.56e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9391s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0074 | max_forward: 0.7478 | max_loss: 0.4124 | max_backward: 1.5474 | max_optimizer_step: 0.1789 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5904.2 | uss_MB: 4869.2 | child_rss_MB: 5389.9 | allocated_MB: 372.1 | reserved_MB: 26836.0 | inactive_split_MB: 275.7 +rank7 [2026-10-29 19:35:33,871] INFO : Running validation on rank7 +rank7 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 19:35:33,872] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 19:36:22,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:36:23,273] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 19:36:50,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 19:36:50,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 19:36:51,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 19:36:52,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 19:37:02,927] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:37:03,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:37:04,354] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 19:37:20,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 19:37:21,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 19:37:22,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 19:37:23,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 19:37:33,325] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:37:33,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:37:34,029] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 19:37:49,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 19:37:49,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 19:37:50,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 19:37:51,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_42000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 19:38:01,091] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 19:42:30,057] INFO : Step 42100/50000 rank7 | Train Loss: 1.2859 | Train Loss Unweighted: 0.6316 | Train Loss Components: Auxiliary: 0.2797 | Cardinality: 0.1585 | Classification: 0.0788 | Classification_binary: 0.1783 | HitGrouping: 0.2382 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1857 | Regression_eta: 0.0016 | Regression_pt: 0.1852 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0906 | Regression_eta: 0.0505 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0649 | LR: 1.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0602s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7033 | max_loss: 0.2430 | max_backward: 1.6521 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0823 | Memory: rss_MB: 7777.3 | uss_MB: 4955.3 | child_rss_MB: 5397.4 | allocated_MB: 371.2 | reserved_MB: 22146.0 | inactive_split_MB: 212.7 +rank7 [2026-10-29 19:46:36,780] INFO : Step 42200/50000 rank7 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6766 | Train Loss Components: Auxiliary: 0.2865 | Cardinality: 0.1311 | Classification: 0.0887 | Classification_binary: 0.1867 | HitGrouping: 0.2425 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1981 | Regression_eta: 0.0016 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0829 | Regression_eta: 0.0497 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0639 | LR: 1.53e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8366s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7972 | max_loss: 0.1838 | max_backward: 1.4548 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 6007.0 | uss_MB: 3874.7 | child_rss_MB: 5415.5 | allocated_MB: 375.1 | reserved_MB: 23140.0 | inactive_split_MB: 272.7 +rank7 [2026-10-29 19:47:23,595] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank7 [2026-10-29 19:47:23,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 19:50:51,512] INFO : Step 42300/50000 rank7 | Train Loss: 1.3237 | Train Loss Unweighted: 0.6731 | Train Loss Components: Auxiliary: 0.2856 | Cardinality: 0.1444 | Classification: 0.0971 | Classification_binary: 0.2107 | HitGrouping: 0.2373 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1802 | Regression_eta: 0.0016 | Regression_pt: 0.1816 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2207 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0845 | Regression_eta: 0.0652 | Regression_pt: 0.0888 | Regression_sin_phi: 0.1227 | LR: 1.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2014s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0619 | max_forward: 0.7966 | max_loss: 0.1497 | max_backward: 1.6272 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.7255 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6036.1 | uss_MB: 5027.7 | child_rss_MB: 5389.1 | allocated_MB: 374.0 | reserved_MB: 24148.0 | inactive_split_MB: 268.0 +rank7 [2026-10-29 19:54:45,040] INFO : Step 42400/50000 rank7 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6472 | Train Loss Components: Auxiliary: 0.2718 | Cardinality: 0.1254 | Classification: 0.0923 | Classification_binary: 0.1654 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1932 | Regression_eta: 0.0016 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0896 | Regression_eta: 0.0628 | Regression_pt: 0.0926 | Regression_sin_phi: 0.1043 | LR: 1.50e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6751s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.6424 | max_loss: 0.2413 | max_backward: 1.4357 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5991.1 | uss_MB: 4983.0 | child_rss_MB: 5399.1 | allocated_MB: 367.9 | reserved_MB: 24150.0 | inactive_split_MB: 229.1 +rank7 [2026-10-29 19:58:48,121] INFO : Step 42500/50000 rank7 | Train Loss: 1.4159 | Train Loss Unweighted: 0.7311 | Train Loss Components: Auxiliary: 0.3061 | Cardinality: 0.1401 | Classification: 0.1046 | Classification_binary: 0.2099 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2053 | Regression_eta: 0.0021 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.2262 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0930 | Regression_eta: 0.0884 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0853 | LR: 1.49e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1955s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2597 | max_backward: 1.6599 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6532 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 5992.0 | uss_MB: 4986.6 | child_rss_MB: 5404.0 | allocated_MB: 372.3 | reserved_MB: 24150.0 | inactive_split_MB: 248.7 +rank7 [2026-10-29 20:02:59,632] INFO : Step 42600/50000 rank7 | Train Loss: 1.4044 | Train Loss Unweighted: 0.7214 | Train Loss Components: Auxiliary: 0.3034 | Cardinality: 0.1514 | Classification: 0.0951 | Classification_binary: 0.2320 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1950 | Regression_eta: 0.0018 | Regression_pt: 0.1952 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2315 | Regression_cos_phi: 0.0976 | Regression_energy: 0.0997 | Regression_eta: 0.0655 | Regression_pt: 0.1027 | Regression_sin_phi: 0.1095 | LR: 1.48e-04 | DataLoad Time: 0.0158s | Model Forward Time: 8.8092s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.2111 | max_backward: 1.5690 | max_optimizer_step: 0.1858 | max_tensorboard_logging: 5.0285 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 5988.9 | uss_MB: 4068.3 | child_rss_MB: 5389.0 | allocated_MB: 373.3 | reserved_MB: 24150.0 | inactive_split_MB: 292.7 +rank7 [2026-10-29 20:06:58,088] INFO : Step 42700/50000 rank7 | Train Loss: 1.2358 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1401 | Classification: 0.0847 | Classification_binary: 0.1675 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1913 | Regression_eta: 0.0016 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0813 | Regression_eta: 0.0499 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0498 | LR: 1.47e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7999s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0646 | max_forward: 0.7310 | max_loss: 0.1485 | max_backward: 1.2811 | max_optimizer_step: 0.1893 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0988 | Memory: rss_MB: 5997.6 | uss_MB: 4989.9 | child_rss_MB: 5398.4 | allocated_MB: 369.1 | reserved_MB: 25176.0 | inactive_split_MB: 216.9 +rank7 [2026-10-29 20:10:49,732] INFO : Step 42800/50000 rank7 | Train Loss: 1.2736 | Train Loss Unweighted: 0.6548 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1161 | Classification: 0.0958 | Classification_binary: 0.1827 | HitGrouping: 0.2422 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1848 | Regression_eta: 0.0017 | Regression_pt: 0.1878 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.1790 | Regression_cos_phi: 0.1217 | Regression_energy: 0.0933 | Regression_eta: 0.0967 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0863 | LR: 1.45e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5856s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0811 | max_forward: 0.7314 | max_loss: 0.1394 | max_backward: 1.3128 | max_optimizer_step: 0.1854 | max_tensorboard_logging: 0.5182 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5998.2 | uss_MB: 4991.4 | child_rss_MB: 5381.5 | allocated_MB: 373.8 | reserved_MB: 25176.0 | inactive_split_MB: 254.2 +rank7 [2026-10-29 20:14:51,336] INFO : Step 42900/50000 rank7 | Train Loss: 1.3069 | Train Loss Unweighted: 0.6732 | Train Loss Components: Auxiliary: 0.2944 | Cardinality: 0.1489 | Classification: 0.0887 | Classification_binary: 0.2182 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1810 | Regression_eta: 0.0015 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0846 | Regression_eta: 0.0508 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0646 | LR: 1.44e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0180 | max_forward: 0.7862 | max_loss: 0.1841 | max_backward: 1.7765 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8743 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5995.3 | uss_MB: 4993.6 | child_rss_MB: 5384.1 | allocated_MB: 375.9 | reserved_MB: 25176.0 | inactive_split_MB: 276.1 +rank7 [2026-10-29 20:18:47,608] INFO : Step 43000/50000 rank7 | Train Loss: 1.2106 | Train Loss Unweighted: 0.6158 | Train Loss Components: Auxiliary: 0.2614 | Cardinality: 0.1317 | Classification: 0.0874 | Classification_binary: 0.1851 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1696 | Regression_eta: 0.0015 | Regression_pt: 0.1705 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0754 | Regression_eta: 0.0472 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0693 | LR: 1.43e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9454s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0150 | max_forward: 0.9739 | max_loss: 0.1984 | max_backward: 1.9954 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.9483 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 5992.6 | uss_MB: 4993.7 | child_rss_MB: 5387.0 | allocated_MB: 372.7 | reserved_MB: 25390.0 | inactive_split_MB: 202.7 +rank7 [2026-10-29 20:18:48,627] INFO : Running validation on rank7 +rank7 [2026-10-29 20:18:48,627] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 20:18:48,628] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 20:19:38,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:19:39,385] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 20:20:09,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 20:20:10,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 20:20:11,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 20:20:12,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 20:20:22,647] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 20:20:23,213] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:20:23,703] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 20:20:39,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 20:20:40,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 20:20:41,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 20:20:42,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 20:20:52,760] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 20:20:53,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:20:53,999] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 20:21:09,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 20:21:09,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 20:21:10,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 20:21:11,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_43000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 20:21:21,241] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 20:25:40,524] INFO : Step 43100/50000 rank7 | Train Loss: 1.1814 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1153 | Classification: 0.0820 | Classification_binary: 0.1617 | HitGrouping: 0.2365 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1726 | Regression_eta: 0.0015 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0814 | Regression_eta: 0.0389 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0586 | LR: 1.42e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.2849s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.6825 | max_loss: 0.1897 | max_backward: 1.4847 | max_optimizer_step: 0.2110 | max_tensorboard_logging: 0.8827 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.1610 | Memory: rss_MB: 7726.4 | uss_MB: 6725.2 | child_rss_MB: 5417.8 | allocated_MB: 369.1 | reserved_MB: 24900.0 | inactive_split_MB: 250.2 +rank7 [2026-10-29 20:29:28,207] INFO : Step 43200/50000 rank7 | Train Loss: 1.3308 | Train Loss Unweighted: 0.6699 | Train Loss Components: Auxiliary: 0.2915 | Cardinality: 0.1345 | Classification: 0.1036 | Classification_binary: 0.2117 | HitGrouping: 0.2308 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1746 | Regression_eta: 0.0020 | Regression_pt: 0.1759 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0790 | Regression_eta: 0.0474 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0651 | LR: 1.40e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1817s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0947 | max_forward: 0.7809 | max_loss: 0.1497 | max_backward: 1.4671 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 0.8606 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6071.0 | uss_MB: 5072.8 | child_rss_MB: 5414.3 | allocated_MB: 373.4 | reserved_MB: 26092.0 | inactive_split_MB: 237.1 +rank7 [2026-10-29 20:33:23,251] INFO : Step 43300/50000 rank7 | Train Loss: 1.3658 | Train Loss Unweighted: 0.6830 | Train Loss Components: Auxiliary: 0.3024 | Cardinality: 0.1749 | Classification: 0.0939 | Classification_binary: 0.2124 | HitGrouping: 0.2375 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1861 | Regression_eta: 0.0017 | Regression_pt: 0.1871 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0811 | Regression_eta: 0.0653 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0768 | LR: 1.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3861s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7813 | max_loss: 0.1843 | max_backward: 1.7981 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.8050 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 6078.4 | uss_MB: 5080.6 | child_rss_MB: 5423.2 | allocated_MB: 372.9 | reserved_MB: 26890.0 | inactive_split_MB: 187.1 +rank7 [2026-10-29 20:37:09,704] INFO : Step 43400/50000 rank7 | Train Loss: 1.1428 | Train Loss Unweighted: 0.5932 | Train Loss Components: Auxiliary: 0.2463 | Cardinality: 0.1126 | Classification: 0.0855 | Classification_binary: 0.1576 | HitGrouping: 0.2327 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1732 | Regression_eta: 0.0013 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0734 | Regression_eta: 0.0391 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0629 | LR: 1.38e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5475s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.5794 | max_loss: 0.2014 | max_backward: 1.1137 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.6984 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6065.1 | uss_MB: 5069.6 | child_rss_MB: 5396.5 | allocated_MB: 368.8 | reserved_MB: 27900.0 | inactive_split_MB: 252.2 +rank7 [2026-10-29 20:40:59,929] INFO : Step 43500/50000 rank7 | Train Loss: 1.2382 | Train Loss Unweighted: 0.6459 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1262 | Classification: 0.0935 | Classification_binary: 0.1920 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0015 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2170 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0930 | Regression_eta: 0.0554 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0687 | LR: 1.37e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1839s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.7196 | max_loss: 0.1497 | max_backward: 1.6138 | max_optimizer_step: 0.2666 | max_tensorboard_logging: 0.7961 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6069.3 | uss_MB: 5079.3 | child_rss_MB: 5422.0 | allocated_MB: 372.7 | reserved_MB: 27900.0 | inactive_split_MB: 270.6 +rank7 [2026-10-29 20:44:45,322] INFO : Step 43600/50000 rank7 | Train Loss: 1.3385 | Train Loss Unweighted: 0.6645 | Train Loss Components: Auxiliary: 0.2924 | Cardinality: 0.1379 | Classification: 0.0987 | Classification_binary: 0.2253 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1677 | Regression_eta: 0.0019 | Regression_pt: 0.1688 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2183 | Regression_cos_phi: 0.1084 | Regression_energy: 0.0806 | Regression_eta: 0.0744 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0868 | LR: 1.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0830s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7764 | max_loss: 0.1536 | max_backward: 1.6237 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5687 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6070.3 | uss_MB: 5081.6 | child_rss_MB: 5400.0 | allocated_MB: 372.5 | reserved_MB: 27900.0 | inactive_split_MB: 262.7 +rank7 [2026-10-29 20:48:25,992] INFO : Step 43700/50000 rank7 | Train Loss: 1.2313 | Train Loss Unweighted: 0.6339 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1293 | Classification: 0.0893 | Classification_binary: 0.1761 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1822 | Regression_eta: 0.0016 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0968 | Regression_eta: 0.0615 | Regression_pt: 0.1008 | Regression_sin_phi: 0.0709 | LR: 1.35e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.4585s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0058 | max_forward: 0.6767 | max_loss: 0.2348 | max_backward: 1.0558 | max_optimizer_step: 0.2062 | max_tensorboard_logging: 0.5774 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6067.4 | uss_MB: 5077.0 | child_rss_MB: 5395.4 | allocated_MB: 368.9 | reserved_MB: 27900.0 | inactive_split_MB: 267.0 +rank7 [2026-10-29 20:52:14,800] INFO : Step 43800/50000 rank7 | Train Loss: 1.2109 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2698 | Cardinality: 0.1178 | Classification: 0.0982 | Classification_binary: 0.1722 | HitGrouping: 0.2312 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1734 | Regression_eta: 0.0016 | Regression_pt: 0.1757 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0873 | Regression_eta: 0.0624 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0482 | LR: 1.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.5356s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0530 | max_forward: 0.6737 | max_loss: 0.1598 | max_backward: 1.2635 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 3.4926 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6064.2 | uss_MB: 5082.0 | child_rss_MB: 5417.3 | allocated_MB: 370.8 | reserved_MB: 27900.0 | inactive_split_MB: 258.3 +rank7 [2026-10-29 20:56:18,488] INFO : Step 43900/50000 rank7 | Train Loss: 1.3549 | Train Loss Unweighted: 0.6842 | Train Loss Components: Auxiliary: 0.2983 | Cardinality: 0.1408 | Classification: 0.1004 | Classification_binary: 0.2203 | HitGrouping: 0.2311 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1786 | Regression_eta: 0.0019 | Regression_pt: 0.1808 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0837 | Regression_eta: 0.0511 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0546 | LR: 1.33e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3478s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8264 | max_loss: 0.1500 | max_backward: 1.5977 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.8559 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6064.4 | uss_MB: 5084.1 | child_rss_MB: 5420.3 | allocated_MB: 372.4 | reserved_MB: 27900.0 | inactive_split_MB: 192.7 +rank7 [2026-10-29 21:00:01,743] INFO : Step 44000/50000 rank7 | Train Loss: 1.2618 | Train Loss Unweighted: 0.6263 | Train Loss Components: Auxiliary: 0.2696 | Cardinality: 0.1397 | Classification: 0.0794 | Classification_binary: 0.1951 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1744 | Regression_eta: 0.0017 | Regression_pt: 0.1737 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0761 | Regression_eta: 0.0451 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0831 | LR: 1.32e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1975s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0063 | max_forward: 0.8502 | max_loss: 0.1677 | max_backward: 1.8503 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.5888 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6066.1 | uss_MB: 5087.4 | child_rss_MB: 5419.6 | allocated_MB: 371.7 | reserved_MB: 28930.0 | inactive_split_MB: 258.8 +rank7 [2026-10-29 21:00:02,713] INFO : Running validation on rank7 +rank7 [2026-10-29 21:00:02,713] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:00:02,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:00:57,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:00:58,022] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 21:01:24,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:01:25,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:01:26,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:01:27,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:01:37,537] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:01:38,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:01:38,955] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 21:01:54,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:01:55,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:01:56,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:01:57,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:02:07,693] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 21:02:08,375] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:02:08,957] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 21:02:24,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:02:24,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:02:25,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:02:26,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_44000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:02:36,090] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 21:06:42,683] INFO : Step 44100/50000 rank7 | Train Loss: 1.1792 | Train Loss Unweighted: 0.5979 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1201 | Classification: 0.0764 | Classification_binary: 0.1596 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1792 | Regression_eta: 0.0015 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0459 | Regression_energy: 0.1155 | Regression_eta: 0.0500 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0588 | LR: 1.31e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.6663 | max_loss: 0.2394 | max_backward: 1.3277 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 1.0388 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 7755.1 | uss_MB: 6776.5 | child_rss_MB: 5415.4 | allocated_MB: 368.7 | reserved_MB: 23252.0 | inactive_split_MB: 215.0 +rank7 [2026-10-29 21:10:40,736] INFO : Step 44200/50000 rank7 | Train Loss: 1.2796 | Train Loss Unweighted: 0.6310 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1540 | Classification: 0.0972 | Classification_binary: 0.1955 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1666 | Regression_eta: 0.0015 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0852 | Regression_eta: 0.0369 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0589 | LR: 1.30e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0064 | max_forward: 0.7424 | max_loss: 0.1450 | max_backward: 1.3222 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.7649 | max_tensorboard_flush: 0.0080 | max_post_step_barrier: 0.1362 | Memory: rss_MB: 6082.4 | uss_MB: 3907.1 | child_rss_MB: 5415.0 | allocated_MB: 374.4 | reserved_MB: 24126.0 | inactive_split_MB: 201.6 +rank7 [2026-10-29 21:14:29,482] INFO : Step 44300/50000 rank7 | Train Loss: 1.2515 | Train Loss Unweighted: 0.6385 | Train Loss Components: Auxiliary: 0.2775 | Cardinality: 0.1363 | Classification: 0.0883 | Classification_binary: 0.2015 | HitGrouping: 0.2248 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1721 | Regression_eta: 0.0015 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0907 | Regression_eta: 0.0411 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0512 | LR: 1.29e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1178s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7644 | max_loss: 0.1924 | max_backward: 1.7044 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.4887 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6087.0 | uss_MB: 5144.4 | child_rss_MB: 5392.6 | allocated_MB: 372.4 | reserved_MB: 24386.0 | inactive_split_MB: 245.0 +rank7 [2026-10-29 21:18:20,341] INFO : Step 44400/50000 rank7 | Train Loss: 1.1246 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1206 | Classification: 0.0737 | Classification_binary: 0.1547 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1650 | Regression_eta: 0.0014 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0743 | Regression_eta: 0.0472 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0572 | LR: 1.28e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7687s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.6869 | max_loss: 0.1812 | max_backward: 1.5974 | max_optimizer_step: 0.1904 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6035.4 | uss_MB: 5145.5 | child_rss_MB: 5391.0 | allocated_MB: 368.7 | reserved_MB: 24386.0 | inactive_split_MB: 188.5 +rank7 [2026-10-29 21:22:16,363] INFO : Step 44500/50000 rank7 | Train Loss: 1.2477 | Train Loss Unweighted: 0.6268 | Train Loss Components: Auxiliary: 0.2656 | Cardinality: 0.1245 | Classification: 0.1001 | Classification_binary: 0.1836 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1704 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0851 | Regression_eta: 0.0675 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0585 | LR: 1.27e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8098s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6709 | max_loss: 0.2404 | max_backward: 1.4673 | max_optimizer_step: 0.1914 | max_tensorboard_logging: 0.6098 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6036.3 | uss_MB: 5146.9 | child_rss_MB: 5412.6 | allocated_MB: 371.6 | reserved_MB: 24386.0 | inactive_split_MB: 155.1 +rank7 [2026-10-29 21:26:10,594] INFO : Step 44600/50000 rank7 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1552 | Classification: 0.0956 | Classification_binary: 0.2160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1792 | Regression_eta: 0.0017 | Regression_pt: 0.1800 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0926 | Regression_eta: 0.0560 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0783 | LR: 1.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1475s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.7877 | max_loss: 0.1530 | max_backward: 1.7015 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6019.3 | uss_MB: 5152.5 | child_rss_MB: 5420.7 | allocated_MB: 372.3 | reserved_MB: 24386.0 | inactive_split_MB: 212.8 +rank7 [2026-10-29 21:30:16,074] INFO : Step 44700/50000 rank7 | Train Loss: 1.0995 | Train Loss Unweighted: 0.5638 | Train Loss Components: Auxiliary: 0.2394 | Cardinality: 0.1054 | Classification: 0.0665 | Classification_binary: 0.1621 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1655 | Regression_eta: 0.0012 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0667 | Regression_eta: 0.0476 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0559 | LR: 1.25e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6980s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.6615 | max_loss: 0.1463 | max_backward: 1.3642 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4930 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6010.0 | uss_MB: 5145.3 | child_rss_MB: 5422.9 | allocated_MB: 367.3 | reserved_MB: 25384.0 | inactive_split_MB: 283.1 +rank7 [2026-10-29 21:34:09,883] INFO : Step 44800/50000 rank7 | Train Loss: 1.1737 | Train Loss Unweighted: 0.5922 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1236 | Classification: 0.0918 | Classification_binary: 0.1650 | HitGrouping: 0.2352 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1656 | Regression_eta: 0.0013 | Regression_pt: 0.1669 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0812 | Regression_eta: 0.0437 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0695 | LR: 1.24e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.4077s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0070 | max_forward: 0.6145 | max_loss: 0.1411 | max_backward: 1.2891 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.4790 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6009.2 | uss_MB: 5150.3 | child_rss_MB: 5416.0 | allocated_MB: 372.0 | reserved_MB: 25384.0 | inactive_split_MB: 133.2 +rank7 [2026-10-29 21:38:11,005] INFO : Step 44900/50000 rank7 | Train Loss: 1.2703 | Train Loss Unweighted: 0.6481 | Train Loss Components: Auxiliary: 0.2859 | Cardinality: 0.1418 | Classification: 0.0955 | Classification_binary: 0.1969 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0839 | Regression_eta: 0.0544 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0667 | LR: 1.23e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9526s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7037 | max_loss: 0.2318 | max_backward: 1.6155 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.5389 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 6014.9 | uss_MB: 5155.3 | child_rss_MB: 5417.4 | allocated_MB: 376.2 | reserved_MB: 25384.0 | inactive_split_MB: 254.3 +rank7 [2026-10-29 21:42:13,638] INFO : Step 45000/50000 rank7 | Train Loss: 1.3238 | Train Loss Unweighted: 0.6669 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.1487 | Classification: 0.0946 | Classification_binary: 0.2071 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1806 | Regression_eta: 0.0018 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0949 | Regression_eta: 0.0441 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0620 | LR: 1.22e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0114s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.7198 | max_loss: 0.1622 | max_backward: 1.7206 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5202 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6016.8 | uss_MB: 5124.1 | child_rss_MB: 5411.6 | allocated_MB: 369.1 | reserved_MB: 25384.0 | inactive_split_MB: 274.9 +rank7 [2026-10-29 21:42:14,559] INFO : Running validation on rank7 +rank7 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:42:14,559] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:43:06,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:43:07,019] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 21:43:32,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:43:33,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:43:34,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:43:35,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:43:45,792] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:43:47,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:43:47,814] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 21:44:03,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:44:04,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:44:05,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:44:06,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:44:16,565] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 21:44:17,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:44:17,710] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 21:44:33,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:44:33,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:44:34,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:44:35,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_45000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:44:45,091] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 21:48:55,616] INFO : Step 45100/50000 rank7 | Train Loss: 1.2002 | Train Loss Unweighted: 0.6171 | Train Loss Components: Auxiliary: 0.2637 | Cardinality: 0.1181 | Classification: 0.0851 | Classification_binary: 0.1564 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1855 | Regression_eta: 0.0017 | Regression_pt: 0.1865 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0675 | Regression_energy: 0.1021 | Regression_eta: 0.0634 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0661 | LR: 1.21e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4086s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0765 | max_forward: 0.7594 | max_loss: 0.1564 | max_backward: 1.3714 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 1.2081 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7627.8 | uss_MB: 6204.9 | child_rss_MB: 5420.2 | allocated_MB: 371.6 | reserved_MB: 24136.0 | inactive_split_MB: 269.2 +rank7 [2026-10-29 21:52:45,386] INFO : Step 45200/50000 rank7 | Train Loss: 1.3111 | Train Loss Unweighted: 0.6779 | Train Loss Components: Auxiliary: 0.2883 | Cardinality: 0.1401 | Classification: 0.1059 | Classification_binary: 0.1977 | HitGrouping: 0.2330 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1842 | Regression_eta: 0.0016 | Regression_pt: 0.1866 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0883 | Regression_eta: 0.0503 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0614 | LR: 1.20e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1518s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7122 | max_loss: 0.1448 | max_backward: 1.4087 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.8860 | max_tensorboard_flush: 0.0044 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7628.2 | uss_MB: 6760.5 | child_rss_MB: 5423.9 | allocated_MB: 373.1 | reserved_MB: 25150.0 | inactive_split_MB: 208.7 +rank7 [2026-10-29 21:56:35,621] INFO : Step 45300/50000 rank7 | Train Loss: 1.3286 | Train Loss Unweighted: 0.6782 | Train Loss Components: Auxiliary: 0.2941 | Cardinality: 0.1535 | Classification: 0.0999 | Classification_binary: 0.2137 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1801 | Regression_eta: 0.0016 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0781 | Regression_eta: 0.0497 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0655 | LR: 1.19e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.0117s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7585 | max_loss: 0.2818 | max_backward: 1.6718 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4828 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6166.0 | uss_MB: 5098.7 | child_rss_MB: 5417.9 | allocated_MB: 377.3 | reserved_MB: 25150.0 | inactive_split_MB: 279.1 +rank7 [2026-10-29 22:00:32,927] INFO : Step 45400/50000 rank7 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5772 | Train Loss Components: Auxiliary: 0.2486 | Cardinality: 0.1346 | Classification: 0.0748 | Classification_binary: 0.1562 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1717 | Regression_eta: 0.0014 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0770 | Regression_eta: 0.0362 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0467 | LR: 1.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9034s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.9405 | max_loss: 0.1469 | max_backward: 1.4783 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 0.6318 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6166.6 | uss_MB: 5303.3 | child_rss_MB: 5404.3 | allocated_MB: 371.1 | reserved_MB: 25150.0 | inactive_split_MB: 273.1 +rank7 [2026-10-29 22:04:23,233] INFO : Step 45500/50000 rank7 | Train Loss: 1.2203 | Train Loss Unweighted: 0.6252 | Train Loss Components: Auxiliary: 0.2714 | Cardinality: 0.1238 | Classification: 0.0956 | Classification_binary: 0.1736 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1753 | Regression_eta: 0.0015 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0804 | Regression_eta: 0.0477 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0809 | LR: 1.18e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6357s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0556 | max_forward: 0.7065 | max_loss: 0.1427 | max_backward: 1.3449 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4800 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6170.8 | uss_MB: 5308.9 | child_rss_MB: 5417.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 202.9 +rank7 [2026-10-29 22:08:21,194] INFO : Step 45600/50000 rank7 | Train Loss: 1.2995 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2841 | Cardinality: 0.1442 | Classification: 0.0912 | Classification_binary: 0.2168 | HitGrouping: 0.2317 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1701 | Regression_eta: 0.0016 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.2276 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0757 | Regression_eta: 0.0449 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0592 | LR: 1.17e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0165s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.7624 | max_loss: 0.1514 | max_backward: 1.6174 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.5956 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5410.8 | allocated_MB: 373.5 | reserved_MB: 26172.0 | inactive_split_MB: 252.9 +rank7 [2026-10-29 22:12:11,369] INFO : Step 45700/50000 rank7 | Train Loss: 1.1987 | Train Loss Unweighted: 0.5991 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1165 | Classification: 0.0880 | Classification_binary: 0.1821 | HitGrouping: 0.2294 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1621 | Regression_eta: 0.0017 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0733 | Regression_eta: 0.0739 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0860 | LR: 1.16e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0838s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0953 | max_forward: 0.8575 | max_loss: 0.1543 | max_backward: 1.5472 | max_optimizer_step: 0.2971 | max_tensorboard_logging: 0.5802 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6168.8 | uss_MB: 5308.7 | child_rss_MB: 5399.4 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 283.8 +rank7 [2026-10-29 22:16:08,602] INFO : Step 45800/50000 rank7 | Train Loss: 1.1325 | Train Loss Unweighted: 0.5755 | Train Loss Components: Auxiliary: 0.2489 | Cardinality: 0.1024 | Classification: 0.0750 | Classification_binary: 0.1554 | HitGrouping: 0.2213 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1693 | Regression_eta: 0.0015 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0784 | Regression_eta: 0.0453 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0543 | LR: 1.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6674s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.6785 | max_loss: 0.1481 | max_backward: 1.4348 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.5293 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6175.1 | uss_MB: 5316.0 | child_rss_MB: 5417.6 | allocated_MB: 370.6 | reserved_MB: 26172.0 | inactive_split_MB: 181.2 +rank7 [2026-10-29 22:20:18,710] INFO : Step 45900/50000 rank7 | Train Loss: 1.2165 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2699 | Cardinality: 0.1433 | Classification: 0.0797 | Classification_binary: 0.1865 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1665 | Regression_eta: 0.0014 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0789 | Regression_eta: 0.0424 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0661 | LR: 1.15e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9620s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.7055 | max_loss: 0.2500 | max_backward: 1.6733 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 0.5437 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6174.5 | uss_MB: 5312.2 | child_rss_MB: 5421.2 | allocated_MB: 371.2 | reserved_MB: 26172.0 | inactive_split_MB: 208.8 +rank7 [2026-10-29 22:24:24,449] INFO : Step 46000/50000 rank7 | Train Loss: 1.2145 | Train Loss Unweighted: 0.6133 | Train Loss Components: Auxiliary: 0.2749 | Cardinality: 0.1379 | Classification: 0.0792 | Classification_binary: 0.1937 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1681 | Regression_eta: 0.0015 | Regression_pt: 0.1693 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0791 | Regression_eta: 0.0562 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0634 | LR: 1.14e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2976s | Phase Times: max_pre_step_barrier: 0.0047 | max_batch_to_device: 0.0834 | max_forward: 0.9739 | max_loss: 0.2430 | max_backward: 1.7322 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.5160 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6177.3 | uss_MB: 5315.1 | child_rss_MB: 5421.5 | allocated_MB: 371.4 | reserved_MB: 26172.0 | inactive_split_MB: 272.2 +rank7 [2026-10-29 22:24:25,404] INFO : Running validation on rank7 +rank7 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 22:24:25,404] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 22:25:16,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:25:17,232] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 22:25:46,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 22:25:47,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 22:25:48,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 22:25:49,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 22:25:59,211] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 22:25:59,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:26:00,492] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 22:26:16,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 22:26:17,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 22:26:18,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 22:26:20,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 22:26:30,058] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 22:26:31,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:26:31,924] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 22:26:47,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 22:26:48,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 22:26:48,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 22:26:49,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_46000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 22:26:59,403] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 22:31:16,603] INFO : Step 46100/50000 rank7 | Train Loss: 1.1285 | Train Loss Unweighted: 0.5852 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1114 | Classification: 0.0849 | Classification_binary: 0.1557 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1703 | Regression_eta: 0.0014 | Regression_pt: 0.1714 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0884 | Regression_eta: 0.0546 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0609 | LR: 1.13e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0428s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.6845 | max_loss: 0.4244 | max_backward: 1.7220 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4999 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 7636.2 | uss_MB: 6769.0 | child_rss_MB: 5402.5 | allocated_MB: 369.4 | reserved_MB: 23490.0 | inactive_split_MB: 178.5 +rank7 [2026-10-29 22:35:12,809] INFO : Step 46200/50000 rank7 | Train Loss: 1.2028 | Train Loss Unweighted: 0.6261 | Train Loss Components: Auxiliary: 0.2655 | Cardinality: 0.1112 | Classification: 0.0911 | Classification_binary: 0.1818 | HitGrouping: 0.2304 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1737 | Regression_eta: 0.0016 | Regression_pt: 0.1762 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0879 | Regression_eta: 0.0755 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0710 | LR: 1.13e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9305s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 1.0096 | max_loss: 0.1442 | max_backward: 1.4170 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7638.8 | uss_MB: 6773.0 | child_rss_MB: 5409.4 | allocated_MB: 369.7 | reserved_MB: 24520.0 | inactive_split_MB: 206.7 +rank7 [2026-10-29 22:39:25,150] INFO : Step 46300/50000 rank7 | Train Loss: 1.2548 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1295 | Classification: 0.0905 | Classification_binary: 0.2037 | HitGrouping: 0.2260 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1722 | Regression_eta: 0.0016 | Regression_pt: 0.1748 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0788 | Regression_eta: 0.0548 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0697 | LR: 1.12e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9851s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7534 | max_loss: 0.2472 | max_backward: 1.6630 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6233.5 | uss_MB: 5367.7 | child_rss_MB: 5418.1 | allocated_MB: 371.7 | reserved_MB: 24520.0 | inactive_split_MB: 276.3 +rank7 [2026-10-29 22:43:23,785] INFO : Step 46400/50000 rank7 | Train Loss: 1.2019 | Train Loss Unweighted: 0.6058 | Train Loss Components: Auxiliary: 0.2661 | Cardinality: 0.1168 | Classification: 0.0741 | Classification_binary: 0.1795 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1736 | Regression_eta: 0.0016 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0837 | Regression_eta: 0.0433 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0501 | LR: 1.11e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6680s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7476 | max_loss: 0.1625 | max_backward: 1.3536 | max_optimizer_step: 0.1964 | max_tensorboard_logging: 1.4212 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0966 | Memory: rss_MB: 6223.4 | uss_MB: 5368.7 | child_rss_MB: 5418.6 | allocated_MB: 372.1 | reserved_MB: 24808.0 | inactive_split_MB: 256.2 +rank7 [2026-10-29 22:47:17,351] INFO : Step 46500/50000 rank7 | Train Loss: 1.1544 | Train Loss Unweighted: 0.5810 | Train Loss Components: Auxiliary: 0.2456 | Cardinality: 0.1178 | Classification: 0.0881 | Classification_binary: 0.1624 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1636 | Regression_eta: 0.0015 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0719 | Regression_eta: 0.0373 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0575 | LR: 1.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5733s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2513 | max_backward: 1.7586 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9991 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6221.5 | uss_MB: 5018.2 | child_rss_MB: 5405.0 | allocated_MB: 371.2 | reserved_MB: 25228.0 | inactive_split_MB: 289.9 +rank7 [2026-10-29 22:51:20,584] INFO : Step 46600/50000 rank7 | Train Loss: 1.2117 | Train Loss Unweighted: 0.6235 | Train Loss Components: Auxiliary: 0.2721 | Cardinality: 0.1301 | Classification: 0.1018 | Classification_binary: 0.1965 | HitGrouping: 0.2240 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1609 | Regression_eta: 0.0014 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2045 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0816 | Regression_eta: 0.0460 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0571 | LR: 1.10e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3977s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0074 | max_forward: 1.0684 | max_loss: 0.4854 | max_backward: 2.6467 | max_optimizer_step: 0.2835 | max_tensorboard_logging: 0.7006 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6221.0 | uss_MB: 5178.5 | child_rss_MB: 5413.6 | allocated_MB: 373.9 | reserved_MB: 25228.0 | inactive_split_MB: 266.5 +rank7 [2026-10-29 22:55:26,213] INFO : Step 46700/50000 rank7 | Train Loss: 1.2108 | Train Loss Unweighted: 0.6104 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1364 | Classification: 0.0849 | Classification_binary: 0.1845 | HitGrouping: 0.2279 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1683 | Regression_eta: 0.0015 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0744 | Regression_eta: 0.0418 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0648 | LR: 1.10e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0469s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7103 | max_loss: 0.2487 | max_backward: 1.5739 | max_optimizer_step: 0.2009 | max_tensorboard_logging: 0.6049 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 6220.6 | uss_MB: 5369.1 | child_rss_MB: 5413.7 | allocated_MB: 373.5 | reserved_MB: 26248.0 | inactive_split_MB: 237.1 +rank7 [2026-10-29 22:59:26,804] INFO : Step 46800/50000 rank7 | Train Loss: 1.1232 | Train Loss Unweighted: 0.5858 | Train Loss Components: Auxiliary: 0.2473 | Cardinality: 0.1043 | Classification: 0.0890 | Classification_binary: 0.1582 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1673 | Regression_eta: 0.0014 | Regression_pt: 0.1683 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0806 | Regression_eta: 0.0578 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0478 | LR: 1.09e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.9120s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0154 | max_forward: 1.6154 | max_loss: 0.4636 | max_backward: 2.3470 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.7114 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6226.3 | uss_MB: 3982.0 | child_rss_MB: 5404.5 | allocated_MB: 370.9 | reserved_MB: 26248.0 | inactive_split_MB: 216.0 +rank7 [2026-10-29 23:04:15,566] INFO : Step 46900/50000 rank7 | Train Loss: 1.2758 | Train Loss Unweighted: 0.6543 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1412 | Classification: 0.0963 | Classification_binary: 0.1895 | HitGrouping: 0.2283 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1817 | Regression_eta: 0.0016 | Regression_pt: 0.1833 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0591 | Regression_energy: 0.1054 | Regression_eta: 0.0463 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0755 | LR: 1.09e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.0393s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7100 | max_loss: 0.2483 | max_backward: 1.6537 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6219.4 | uss_MB: 5373.5 | child_rss_MB: 5409.7 | allocated_MB: 372.1 | reserved_MB: 26248.0 | inactive_split_MB: 298.7 +rank7 [2026-10-29 23:08:15,439] INFO : Step 47000/50000 rank7 | Train Loss: 1.2560 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2761 | Cardinality: 0.1374 | Classification: 0.0886 | Classification_binary: 0.1946 | HitGrouping: 0.2293 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1629 | Regression_eta: 0.0018 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0734 | Regression_eta: 0.0594 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0813 | LR: 1.08e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2715s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.8216 | max_loss: 0.1838 | max_backward: 1.7514 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 6210.1 | uss_MB: 5370.3 | child_rss_MB: 5407.5 | allocated_MB: 371.5 | reserved_MB: 26248.0 | inactive_split_MB: 277.0 +rank7 [2026-10-29 23:08:16,507] INFO : Running validation on rank7 +rank7 [2026-10-29 23:08:16,507] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 23:08:16,508] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 23:09:07,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:09:08,484] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 23:09:36,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 23:09:38,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 23:09:40,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 23:09:42,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 23:09:52,876] INFO : Time taken to make predictions on device 7 is: 0.32 min +rank7 [2026-10-29 23:09:55,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:09:55,736] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 23:10:11,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 23:10:12,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 23:10:13,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 23:10:14,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 23:10:24,693] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 23:10:25,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:10:25,936] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 23:10:41,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 23:10:41,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 23:10:42,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 23:10:43,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_47000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 23:10:53,102] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 23:15:13,933] INFO : Step 47100/50000 rank7 | Train Loss: 1.2006 | Train Loss Unweighted: 0.6097 | Train Loss Components: Auxiliary: 0.2615 | Cardinality: 0.1215 | Classification: 0.0875 | Classification_binary: 0.1632 | HitGrouping: 0.2280 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1777 | Regression_eta: 0.0016 | Regression_pt: 0.1779 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0922 | Regression_eta: 0.0403 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0508 | LR: 1.07e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7047s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.6649 | max_loss: 0.1807 | max_backward: 1.2578 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.5337 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0601 | Memory: rss_MB: 7628.9 | uss_MB: 6793.0 | child_rss_MB: 5410.6 | allocated_MB: 371.6 | reserved_MB: 22484.0 | inactive_split_MB: 273.8 +rank7 [2026-10-29 23:19:19,773] INFO : Step 47200/50000 rank7 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5779 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1178 | Classification: 0.1020 | Classification_binary: 0.1736 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1487 | Regression_eta: 0.0015 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0643 | Regression_eta: 0.0398 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0604 | LR: 1.07e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.6706s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0105 | max_forward: 0.7649 | max_loss: 0.1440 | max_backward: 2.5706 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6563 | max_tensorboard_flush: 0.0021 | max_post_step_barrier: 0.0660 | Memory: rss_MB: 7627.4 | uss_MB: 6364.6 | child_rss_MB: 5413.4 | allocated_MB: 373.7 | reserved_MB: 24508.0 | inactive_split_MB: 234.6 +rank7 [2026-10-29 23:23:31,491] INFO : Step 47300/50000 rank7 | Train Loss: 1.3803 | Train Loss Unweighted: 0.6815 | Train Loss Components: Auxiliary: 0.2973 | Cardinality: 0.1625 | Classification: 0.0981 | Classification_binary: 0.2228 | HitGrouping: 0.2385 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1773 | Regression_eta: 0.0018 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0808 | Regression_eta: 0.0698 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0837 | LR: 1.06e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7150s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7643 | max_loss: 0.4237 | max_backward: 1.6634 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 1.1888 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 6305.1 | uss_MB: 5487.9 | child_rss_MB: 5410.0 | allocated_MB: 371.2 | reserved_MB: 24508.0 | inactive_split_MB: 266.8 +rank7 [2026-10-29 23:27:29,601] INFO : Step 47400/50000 rank7 | Train Loss: 1.1822 | Train Loss Unweighted: 0.5933 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1277 | Classification: 0.0795 | Classification_binary: 0.1741 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1674 | Regression_eta: 0.0015 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0834 | Regression_eta: 0.0616 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0767 | LR: 1.06e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0194s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7379 | max_loss: 0.1499 | max_backward: 1.3988 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6290.6 | uss_MB: 5487.5 | child_rss_MB: 5400.7 | allocated_MB: 369.0 | reserved_MB: 24508.0 | inactive_split_MB: 276.8 +rank7 [2026-10-29 23:31:38,582] INFO : Step 47500/50000 rank7 | Train Loss: 1.1191 | Train Loss Unweighted: 0.5821 | Train Loss Components: Auxiliary: 0.2443 | Cardinality: 0.1066 | Classification: 0.0889 | Classification_binary: 0.1522 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1683 | Regression_eta: 0.0014 | Regression_pt: 0.1698 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0924 | Regression_eta: 0.0420 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0448 | LR: 1.06e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8365s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.5916 | max_loss: 0.2355 | max_backward: 1.2295 | max_optimizer_step: 0.2742 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6282.1 | uss_MB: 5489.6 | child_rss_MB: 5393.5 | allocated_MB: 370.1 | reserved_MB: 24508.0 | inactive_split_MB: 229.9 +rank7 [2026-10-29 23:35:34,355] INFO : Step 47600/50000 rank7 | Train Loss: 1.3167 | Train Loss Unweighted: 0.6583 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1266 | Classification: 0.0975 | Classification_binary: 0.2156 | HitGrouping: 0.2433 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1703 | Regression_eta: 0.0016 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0786 | Regression_eta: 0.0562 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0717 | LR: 1.05e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.5018s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7713 | max_loss: 0.1587 | max_backward: 1.7946 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.9061 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6256.8 | uss_MB: 5493.7 | child_rss_MB: 5403.2 | allocated_MB: 372.4 | reserved_MB: 24508.0 | inactive_split_MB: 287.0 +rank7 [2026-10-29 23:39:49,205] INFO : Step 47700/50000 rank7 | Train Loss: 1.2914 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1505 | Classification: 0.0952 | Classification_binary: 0.1980 | HitGrouping: 0.2329 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1690 | Regression_eta: 0.0017 | Regression_pt: 0.1702 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0782 | Regression_eta: 0.0484 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0739 | LR: 1.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8404s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0066 | max_forward: 0.7487 | max_loss: 0.2529 | max_backward: 1.7930 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.9916 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6249.7 | uss_MB: 5494.4 | child_rss_MB: 5400.6 | allocated_MB: 375.6 | reserved_MB: 24876.0 | inactive_split_MB: 191.4 +rank7 [2026-10-29 23:43:53,969] INFO : Step 47800/50000 rank7 | Train Loss: 1.0900 | Train Loss Unweighted: 0.5581 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.0891 | Classification: 0.0818 | Classification_binary: 0.1549 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1588 | Regression_eta: 0.0014 | Regression_pt: 0.1596 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0799 | Regression_eta: 0.0342 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0491 | LR: 1.04e-04 | DataLoad Time: 0.0128s | Model Forward Time: 4.5875s | Phase Times: max_pre_step_barrier: 0.0436 | max_batch_to_device: 0.0111 | max_forward: 0.8340 | max_loss: 0.2298 | max_backward: 1.8771 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.8299 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 6230.6 | uss_MB: 5494.7 | child_rss_MB: 5407.8 | allocated_MB: 369.2 | reserved_MB: 24876.0 | inactive_split_MB: 256.4 +rank7 [2026-10-29 23:47:48,794] INFO : Step 47900/50000 rank7 | Train Loss: 1.3016 | Train Loss Unweighted: 0.6353 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.1464 | Classification: 0.0964 | Classification_binary: 0.1981 | HitGrouping: 0.2344 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1674 | Regression_eta: 0.0018 | Regression_pt: 0.1695 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0695 | Regression_eta: 0.0584 | Regression_pt: 0.0725 | Regression_sin_phi: 0.1089 | LR: 1.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1963s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.6760 | max_loss: 0.1467 | max_backward: 1.4425 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9733 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6221.2 | uss_MB: 5492.7 | child_rss_MB: 5398.4 | allocated_MB: 372.4 | reserved_MB: 25540.0 | inactive_split_MB: 207.1 +rank7 [2026-10-29 23:51:38,726] INFO : Step 48000/50000 rank7 | Train Loss: 1.2878 | Train Loss Unweighted: 0.6389 | Train Loss Components: Auxiliary: 0.2876 | Cardinality: 0.1411 | Classification: 0.0891 | Classification_binary: 0.2092 | HitGrouping: 0.2297 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1682 | Regression_eta: 0.0017 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0835 | Regression_eta: 0.0526 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0615 | LR: 1.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2953s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8072 | max_loss: 0.1539 | max_backward: 1.8412 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6215.2 | uss_MB: 5499.1 | child_rss_MB: 5397.6 | allocated_MB: 373.1 | reserved_MB: 25540.0 | inactive_split_MB: 286.9 +rank7 [2026-10-29 23:51:39,733] INFO : Running validation on rank7 +rank7 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 23:51:39,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 23:52:31,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:52:31,694] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 23:52:58,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 23:52:59,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 23:53:01,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 23:53:02,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 23:53:12,423] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-29 23:53:13,270] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:53:13,761] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 23:53:29,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 23:53:30,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 23:53:31,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 23:53:32,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 23:53:42,416] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 23:53:43,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:53:43,872] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 23:53:59,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 23:53:59,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 23:54:00,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 23:54:00,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_48000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 23:54:10,956] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-30 00:00:53,531] INFO : Step 48100/50000 rank7 | Train Loss: 1.1868 | Train Loss Unweighted: 0.5997 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1161 | Classification: 0.0850 | Classification_binary: 0.1700 | HitGrouping: 0.2358 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1702 | Regression_eta: 0.0016 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0840 | Regression_eta: 0.0540 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0765 | LR: 1.03e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5286s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0145 | max_forward: 0.7876 | max_loss: 0.2363 | max_backward: 1.3859 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 1.2086 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 7593.0 | uss_MB: 5982.7 | child_rss_MB: 5376.9 | allocated_MB: 370.7 | reserved_MB: 25814.0 | inactive_split_MB: 193.3 +rank7 [2026-10-30 00:04:40,583] INFO : Step 48200/50000 rank7 | Train Loss: 1.1675 | Train Loss Unweighted: 0.5948 | Train Loss Components: Auxiliary: 0.2507 | Cardinality: 0.1089 | Classification: 0.0899 | Classification_binary: 0.1656 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1676 | Regression_eta: 0.0016 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0825 | Regression_eta: 0.0763 | Regression_pt: 0.0850 | Regression_sin_phi: 0.1047 | LR: 1.03e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0272s | Phase Times: max_pre_step_barrier: 0.1083 | max_batch_to_device: 0.0069 | max_forward: 0.6482 | max_loss: 0.1442 | max_backward: 1.1466 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9614 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 7578.5 | uss_MB: 6872.8 | child_rss_MB: 5393.6 | allocated_MB: 373.0 | reserved_MB: 25814.0 | inactive_split_MB: 231.4 +rank7 [2026-10-30 00:08:31,628] INFO : Step 48300/50000 rank7 | Train Loss: 1.2971 | Train Loss Unweighted: 0.6553 | Train Loss Components: Auxiliary: 0.2886 | Cardinality: 0.1443 | Classification: 0.0995 | Classification_binary: 0.2087 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1710 | Regression_eta: 0.0015 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0864 | Regression_eta: 0.0604 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0862 | LR: 1.03e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9458s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.7645 | max_loss: 0.2669 | max_backward: 1.8944 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 1.0376 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6288.5 | uss_MB: 5443.8 | child_rss_MB: 5393.3 | allocated_MB: 375.6 | reserved_MB: 25814.0 | inactive_split_MB: 280.6 +rank7 [2026-10-30 00:12:19,574] INFO : Step 48400/50000 rank7 | Train Loss: 1.2067 | Train Loss Unweighted: 0.5937 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1201 | Classification: 0.0878 | Classification_binary: 0.1811 | HitGrouping: 0.2276 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1596 | Regression_eta: 0.0017 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0640 | Regression_eta: 0.0490 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0377 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3332s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0816 | max_forward: 0.7733 | max_loss: 0.1459 | max_backward: 1.4760 | max_optimizer_step: 0.2981 | max_tensorboard_logging: 0.8703 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6280.8 | uss_MB: 5580.1 | child_rss_MB: 5399.9 | allocated_MB: 372.3 | reserved_MB: 25816.0 | inactive_split_MB: 193.7 +rank7 [2026-10-30 00:16:01,186] INFO : Step 48500/50000 rank7 | Train Loss: 1.1957 | Train Loss Unweighted: 0.5853 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1348 | Classification: 0.0907 | Classification_binary: 0.1625 | HitGrouping: 0.2322 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1644 | Regression_eta: 0.0015 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1576 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0773 | Regression_eta: 0.0311 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0513 | LR: 1.02e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.5860s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0064 | max_forward: 0.6730 | max_loss: 0.1805 | max_backward: 1.1889 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.6281 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6256.8 | uss_MB: 5163.0 | child_rss_MB: 5398.1 | allocated_MB: 371.2 | reserved_MB: 25816.0 | inactive_split_MB: 223.6 +rank7 [2026-10-30 00:19:50,481] INFO : Step 48600/50000 rank7 | Train Loss: 1.2599 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1297 | Classification: 0.1114 | Classification_binary: 0.2024 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1584 | Regression_eta: 0.0016 | Regression_pt: 0.1593 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0802 | Regression_eta: 0.0562 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0597 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9688s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7399 | max_loss: 0.2221 | max_backward: 1.5959 | max_optimizer_step: 0.1979 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0415 | Memory: rss_MB: 6237.7 | uss_MB: 5583.9 | child_rss_MB: 5391.4 | allocated_MB: 369.6 | reserved_MB: 25816.0 | inactive_split_MB: 277.6 +rank7 [2026-10-30 00:23:38,552] INFO : Step 48700/50000 rank7 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2772 | Cardinality: 0.1286 | Classification: 0.0953 | Classification_binary: 0.2080 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1613 | Regression_eta: 0.0015 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0805 | Regression_eta: 0.0405 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0540 | LR: 1.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0936s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0065 | max_forward: 0.7580 | max_loss: 0.1906 | max_backward: 1.5298 | max_optimizer_step: 0.1975 | max_tensorboard_logging: 0.5960 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 6228.9 | uss_MB: 4123.5 | child_rss_MB: 5394.2 | allocated_MB: 371.6 | reserved_MB: 25818.0 | inactive_split_MB: 255.6 +rank7 [2026-10-30 00:27:30,341] INFO : Step 48800/50000 rank7 | Train Loss: 1.1306 | Train Loss Unweighted: 0.5616 | Train Loss Components: Auxiliary: 0.2491 | Cardinality: 0.1292 | Classification: 0.0715 | Classification_binary: 0.1553 | HitGrouping: 0.2278 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1653 | Regression_eta: 0.0014 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0929 | Regression_eta: 0.0543 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0418 | LR: 1.01e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4225s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.6536 | max_loss: 0.1441 | max_backward: 1.9174 | max_optimizer_step: 0.1799 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6231.0 | uss_MB: 5585.9 | child_rss_MB: 5369.9 | allocated_MB: 367.0 | reserved_MB: 25818.0 | inactive_split_MB: 207.2 +rank7 [2026-10-30 00:31:24,394] INFO : Step 48900/50000 rank7 | Train Loss: 1.2420 | Train Loss Unweighted: 0.6022 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1319 | Classification: 0.0991 | Classification_binary: 0.1800 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1593 | Regression_eta: 0.0017 | Regression_pt: 0.1601 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0750 | Regression_eta: 0.0574 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0701 | LR: 1.01e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0144 | max_forward: 1.2726 | max_loss: 0.3364 | max_backward: 2.1384 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.5542 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6219.9 | uss_MB: 5588.1 | child_rss_MB: 5392.9 | allocated_MB: 374.6 | reserved_MB: 25818.0 | inactive_split_MB: 226.5 +rank7 [2026-10-30 00:35:14,113] INFO : Step 49000/50000 rank7 | Train Loss: 1.2892 | Train Loss Unweighted: 0.6428 | Train Loss Components: Auxiliary: 0.2867 | Cardinality: 0.1384 | Classification: 0.0934 | Classification_binary: 0.2164 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1642 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2162 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0928 | Regression_eta: 0.0698 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0681 | LR: 1.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5981s | Phase Times: max_pre_step_barrier: 0.0218 | max_batch_to_device: 0.0152 | max_forward: 1.1973 | max_loss: 0.2254 | max_backward: 2.7260 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.7692 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 6216.7 | uss_MB: 5590.9 | child_rss_MB: 5395.3 | allocated_MB: 369.7 | reserved_MB: 26834.0 | inactive_split_MB: 280.7 +rank7 [2026-10-30 00:35:15,047] INFO : Running validation on rank7 +rank7 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:35:15,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:36:03,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:36:04,036] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 00:36:29,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:36:30,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:36:32,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:36:33,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:36:43,249] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:36:48,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:36:48,746] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 00:37:04,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:37:05,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:37:06,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:37:07,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:37:17,824] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 00:37:17,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:37:18,485] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 00:37:33,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 00:37:34,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 00:37:34,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 00:37:35,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_49000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 00:37:45,600] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-30 00:41:49,929] INFO : Step 49100/50000 rank7 | Train Loss: 1.1701 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1264 | Classification: 0.0759 | Classification_binary: 0.1654 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1730 | Regression_eta: 0.0014 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0876 | Regression_eta: 0.0371 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0687 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6168s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6930 | max_loss: 0.1440 | max_backward: 1.2802 | max_optimizer_step: 0.2123 | max_tensorboard_logging: 0.4965 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 7410.7 | uss_MB: 6780.5 | child_rss_MB: 5371.4 | allocated_MB: 371.6 | reserved_MB: 23242.0 | inactive_split_MB: 267.0 +rank7 [2026-10-30 00:45:59,115] INFO : Step 49200/50000 rank7 | Train Loss: 1.0853 | Train Loss Unweighted: 0.5403 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1074 | Classification: 0.0848 | Classification_binary: 0.1547 | HitGrouping: 0.2268 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1487 | Regression_eta: 0.0013 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0665 | Regression_eta: 0.0351 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0460 | LR: 1.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5905s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.6427 | max_loss: 0.2377 | max_backward: 1.1264 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6832 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7411.5 | uss_MB: 6779.7 | child_rss_MB: 5387.8 | allocated_MB: 369.3 | reserved_MB: 24624.0 | inactive_split_MB: 207.0 +rank7 [2026-10-30 00:49:54,420] INFO : Step 49300/50000 rank7 | Train Loss: 1.2674 | Train Loss Unweighted: 0.6451 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1279 | Classification: 0.1003 | Classification_binary: 0.2000 | HitGrouping: 0.2319 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1705 | Regression_eta: 0.0017 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0744 | Regression_eta: 0.0487 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0469 | LR: 1.00e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.9869s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 1.0117 | max_loss: 0.2078 | max_backward: 2.1281 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6295.3 | uss_MB: 5670.7 | child_rss_MB: 5395.5 | allocated_MB: 371.6 | reserved_MB: 24624.0 | inactive_split_MB: 256.9 +rank7 [2026-10-30 00:53:44,253] INFO : Step 49400/50000 rank7 | Train Loss: 1.2103 | Train Loss Unweighted: 0.6125 | Train Loss Components: Auxiliary: 0.2632 | Cardinality: 0.1160 | Classification: 0.0842 | Classification_binary: 0.1848 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1695 | Regression_eta: 0.0015 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0814 | Regression_eta: 0.0569 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0614 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9619s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.9743 | max_loss: 0.2367 | max_backward: 1.6293 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6295.0 | uss_MB: 5671.0 | child_rss_MB: 5390.3 | allocated_MB: 373.4 | reserved_MB: 25626.0 | inactive_split_MB: 256.6 +rank7 [2026-10-30 00:57:39,408] INFO : Step 49500/50000 rank7 | Train Loss: 1.0931 | Train Loss Unweighted: 0.5501 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1159 | Classification: 0.0729 | Classification_binary: 0.1500 | HitGrouping: 0.2255 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1620 | Regression_eta: 0.0014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0257 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0874 | Regression_eta: 0.0447 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0402 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0105s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7515 | max_loss: 0.4122 | max_backward: 1.3090 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8291 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1079 | Memory: rss_MB: 6290.9 | uss_MB: 5667.1 | child_rss_MB: 5386.7 | allocated_MB: 368.4 | reserved_MB: 25626.0 | inactive_split_MB: 240.7 +rank7 [2026-10-30 01:01:39,140] INFO : Step 49600/50000 rank7 | Train Loss: 1.2332 | Train Loss Unweighted: 0.6152 | Train Loss Components: Auxiliary: 0.2703 | Cardinality: 0.1344 | Classification: 0.1021 | Classification_binary: 0.1955 | HitGrouping: 0.2271 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1568 | Regression_eta: 0.0015 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0813 | Regression_eta: 0.0641 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0418 | LR: 1.00e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8018s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.9814 | max_loss: 0.2009 | max_backward: 1.9495 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.5473 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6297.6 | uss_MB: 5674.2 | child_rss_MB: 5389.7 | allocated_MB: 372.4 | reserved_MB: 25626.0 | inactive_split_MB: 192.9 +rank7 [2026-10-30 01:05:30,841] INFO : Step 49700/50000 rank7 | Train Loss: 1.2569 | Train Loss Unweighted: 0.6141 | Train Loss Components: Auxiliary: 0.2801 | Cardinality: 0.1477 | Classification: 0.0878 | Classification_binary: 0.1957 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1633 | Regression_eta: 0.0016 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0796 | Regression_eta: 0.0752 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0839 | LR: 1.00e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4592s | Phase Times: max_pre_step_barrier: 0.0110 | max_batch_to_device: 0.0069 | max_forward: 0.7662 | max_loss: 0.1753 | max_backward: 1.5943 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.9945 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6295.4 | uss_MB: 5676.2 | child_rss_MB: 5387.5 | allocated_MB: 372.6 | reserved_MB: 25626.0 | inactive_split_MB: 276.1 +rank7 [2026-10-30 01:09:21,932] INFO : Step 49800/50000 rank7 | Train Loss: 1.0991 | Train Loss Unweighted: 0.5513 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.0915 | Classification: 0.0805 | Classification_binary: 0.1537 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0762 | Regression_eta: 0.0360 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0489 | LR: 1.00e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3610s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6613 | max_loss: 0.1518 | max_backward: 1.5006 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1262 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0297 | Memory: rss_MB: 6294.3 | uss_MB: 5674.9 | child_rss_MB: 5380.9 | allocated_MB: 367.3 | reserved_MB: 25626.0 | inactive_split_MB: 206.7 +rank7 [2026-10-30 01:13:13,492] INFO : Step 49900/50000 rank7 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2777 | Cardinality: 0.1432 | Classification: 0.0873 | Classification_binary: 0.1822 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1733 | Regression_eta: 0.0016 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1731 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0888 | Regression_eta: 0.0487 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0668 | LR: 1.00e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0821s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7334 | max_loss: 0.1908 | max_backward: 1.5406 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.7302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1283 | Memory: rss_MB: 6297.1 | uss_MB: 5677.4 | child_rss_MB: 5386.2 | allocated_MB: 373.5 | reserved_MB: 25626.0 | inactive_split_MB: 245.9 +rank7 [2026-10-30 01:17:08,641] INFO : Step 50000/50000 rank7 | Train Loss: 1.3382 | Train Loss Unweighted: 0.6395 | Train Loss Components: Auxiliary: 0.2936 | Cardinality: 0.1730 | Classification: 0.0929 | Classification_binary: 0.2148 | HitGrouping: 0.2364 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1638 | Regression_eta: 0.0017 | Regression_pt: 0.1643 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5907 | Classification_binary: 1.0000 | Regression_cos_phi: 76.8677 | Regression_energy: 0.4836 | Regression_eta: 34.1723 | Regression_pt: 0.5004 | Regression_sin_phi: 76.3305 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2087 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0831 | Regression_eta: 0.0447 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0562 | LR: 1.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8278s | Phase Times: max_pre_step_barrier: 0.1124 | max_batch_to_device: 0.0073 | max_forward: 0.8379 | max_loss: 0.2574 | max_backward: 1.8392 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6298.7 | uss_MB: 5682.1 | child_rss_MB: 5383.0 | allocated_MB: 372.7 | reserved_MB: 25626.0 | inactive_split_MB: 277.0 +rank7 [2026-10-30 01:17:09,619] INFO : Running validation on rank7 +rank7 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 01:17:09,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 01:18:02,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:18:03,255] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 01:18:29,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 01:18:30,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 01:18:31,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 01:18:31,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 01:18:42,002] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:18:42,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:18:43,093] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 01:18:59,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 01:19:00,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 01:19:00,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 01:19:01,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 01:19:11,962] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:19:12,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:19:13,325] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 01:19:28,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:19:29,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:19:29,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:19:30,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_attention_seed12345_20261028_111202_121851/preds_step_50000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:19:40,355] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-30 01:19:41,597] INFO : Training completed. Total time on device 7: 629.368min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-50000.pth b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..25dd48433a5a45ba437484a06dcd3eb322517ea5 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00392bfbe6a32174783b909254db2ed09959cb988650c35d48fd29d51e5dbf75 +size 153793740 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_1000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..d00a506848597a5ddae580e86005049f99c3e44d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.04737540269092287, "train": {"Classification_binary": 0.40864166617393494, "Cardinality": 0.2766936719417572, "Classification": 0.31900978088378906, "Regression_pt": 0.49511682987213135, "Regression_eta": 0.0053130267187952995, "Regression_sin_phi": 0.0026397407054901123, "Regression_cos_phi": 0.0024432295467704535, "Regression_energy": 0.49632391333580017, "HitGrouping": 0.29039323329925537, "Auxiliary": 0.5722727179527283, "Total": 2.791963577270508, "Total_unweighted": 1.7294881872367114}, "train_group": {}, "valid": {"Classification_binary": 0.42940497398376465, "Cardinality": 0.3237307071685791, "Classification": 0.3533770640691121, "Regression_pt": 0.5011221170425415, "Regression_eta": 0.006496857230861981, "Regression_sin_phi": 0.0031525793795784316, "Regression_cos_phi": 0.0030794494474927583, "Regression_energy": 0.49707237879435223, "HitGrouping": 0.2881291111310323, "Auxiliary": 0.5881980657577515, "Total": 3.016747792561849, "Total_unweighted": 1.7937054199477036, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8012059330940247, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.14073704183101654, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2725040018558502, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.13838930428028107, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.13413399457931519, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.48474544286727905, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7593717575073242, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.1420280486345291, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.24906811118125916, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.13997803628444672, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.1332186460494995, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5350728631019592, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.798824667930603, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.16374284029006958, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2801363468170166, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.16166338324546814, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.1563490778207779, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4908067286014557, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7960522174835205, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.15439197421073914, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2748936116695404, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.15222886204719543, "metrics/hit_grouping/calorimeter/all/object_purity": 0.14712569117546082, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.49282675981521606, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025604132562875748, "metrics/hit_grouping/calorimeter/background_recall": 0.6242281198501587, "metrics/hit_grouping/calorimeter/completeness": 0.2559550404548645, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2930687367916107, "metrics/hit_grouping/calorimeter/energy_purity": 0.28978222608566284, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5601152181625366, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8161379098892212, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.1398112177848816, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2039414793252945, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.13920684158802032, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.14292854070663452, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4437742531299591, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7844141721725464, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.172218456864357, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2968219220638275, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.1690102219581604, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.16312654316425323, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.523410975933075, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7656649351119995, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.16667787730693817, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.40222474932670593, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.16230440139770508, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.13933536410331726, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5647832155227661, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7624170184135437, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.1805071383714676, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.35029587149620056, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.17764514684677124, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.1756199598312378, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5734038949012756, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7398192286491394, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.38624653220176697, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.633176326751709, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3755168318748474, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3369487524032593, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6127681732177734, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8394078612327576, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.09664774686098099, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.14798445999622345, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.0963321179151535, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.09220220893621445, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.39085307717323303, "metrics/hit_grouping/calorimeter/purity": 0.25213730335235596, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8886746168136597, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.026669451966881752, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0923076942563057, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.026179490610957146, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.003017076989635825, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8594067096710205, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.030014248564839363, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0672748014330864, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.030298540368676186, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.016289997845888138, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2759407162666321, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7904340028762817, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.16426798701286316, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2907266616821289, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.16192840039730072, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.1577223688364029, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5149855613708496, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6156902313232422, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.34384340047836304, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.17240530252456665, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.17017221450805664, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7643454670906067, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7269093990325928, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3426808714866638, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.2482837736606598, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.19515247642993927, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5771172046661377, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5494126081466675, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.3304717540740967, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.1417613923549652, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.15094874799251556, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.8138930201530457, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5853093862533569, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.33572643995285034, "metrics/hit_grouping/tracker/all/object_completeness": 0.1604180634021759, "metrics/hit_grouping/tracker/all/object_purity": 0.1607474982738495, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7782920598983765, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0018361663678660989, "metrics/hit_grouping/tracker/background_recall": 0.007818515412509441, "metrics/hit_grouping/tracker/completeness": 0.16991452872753143, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5688631534576416, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.5698518753051758, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.32273349165916443, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.1799490749835968, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.20177587866783142, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.8029156923294067, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.583850085735321, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.32933562994003296, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.14023539423942566, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.12695373594760895, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7848848104476929, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6211735606193542, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3713592290878296, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1385059803724289, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.10557594150304794, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7166262269020081, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6103825569152832, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.3675413429737091, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.16607777774333954, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.15270042419433594, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7632997632026672, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6464709639549255, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4607190489768982, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.231863334774971, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.22498396039009094, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6990678906440735, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5418910980224609, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.2723015546798706, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.14048215746879578, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.15993554890155792, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.8117536902427673, "metrics/hit_grouping/tracker/purity": 0.1614612489938736, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8930638432502747, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015638574957847595, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.013703237287700176, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.011583535000681877, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08557775616645813, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6577268242835999, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1301192343235016, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.06894117593765259, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.05386638268828392, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5837221145629883, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5352447628974915, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4026326537132263, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.1907702535390854, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.1931864619255066, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.894575297832489, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.38556256890296936, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6749999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.27601349353790283, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.2727811336517334, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5154770612716675, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.10016476362943649, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.015327702276408672, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.338563919067383, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.14639347791671753, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.015953654423356056, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.3595963716506958, "metrics/particle/class_1/pid_accuracy": 0.8103334307670593, "metrics/particle/class_2/efficiency": 0.3198339641094208, "metrics/particle/class_2/pid_accuracy": 0.050859346985816956, "metrics/particle/class_3/efficiency": 0.4007251262664795, "metrics/particle/class_3/pid_accuracy": 0.8624658584594727, "metrics/particle/class_4/efficiency": 0.36381134390830994, "metrics/particle/class_4/pid_accuracy": 0.0026455025654286146, "metrics/particle/class_5/efficiency": 0.28999999165534973, "metrics/particle/class_5/pid_accuracy": 0.006896551698446274, "metrics/particle/count/bias_mean": -10.044270515441895, "metrics/particle/count/mae": 17.22786521911621, "metrics/particle/count/prediction_mean": 60.01171875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.20605109632015228, "metrics/particle/event/energy_response_mean": 0.9483130574226379, "metrics/particle/event/met_abs_error": 19.504981994628906, "metrics/particle/event/scalar_pt_relative_abs_error": 0.18537193536758423, "metrics/particle/event/scalar_pt_response_mean": 0.934697151184082, "metrics/particle/event/vector_pt_closure": 0.14171867072582245, "metrics/particle/matched/delta_eta_abs_mean": 0.02724774368107319, "metrics/particle/matched/delta_phi_abs_mean": 0.023923629894852638, "metrics/particle/matched/delta_r_mean": 0.040449149906635284, "metrics/particle/matched/energy_relative_abs_mean": 0.22301945090293884, "metrics/particle/matched/pid_accuracy": 0.770228922367096, "metrics/particle/matched/pt_relative_abs_mean": 0.20828650891780853, "metrics/particle/matching/duplicate_fraction": 0.10202000290155411, "metrics/particle/matching/efficiency": 0.37383604049682617, "metrics/particle/matching/f1": 0.4027049243450165, "metrics/particle/matching/purity": 0.43640565872192383, "metrics/particle/matching/target_coverage_dr0p05": 0.3008754253387451, "metrics/particle/matching/target_coverage_dr0p10": 0.45637789368629456, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.23310038447380066, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.4622604548931122, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03958351910114288, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.22368580102920532, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20572999119758606, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.014658715575933456, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2466682493686676, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.02096949890255928, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.5699020624160767, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07294977456331253, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.531282663345337, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 3672.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8777306079864502, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.2857236862182617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.9150078296661377, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.7058558464050293, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.43443742394447327, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.106658935546875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04292271286249161, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.497750997543335, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12518464028835297, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.470445156097412, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8061.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.335771769285202, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.66267466545105, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 24.180837631225586, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 10.439556121826172, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.2362871170043945, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5683339238166809, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 20.847530364990234, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6596126556396484, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 36.99321365356445, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.042225830256938934, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.24951882660388947, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.22531907260417938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.004295556340366602, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.26559916138648987, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.06461538374423981, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 3.4626896381378174, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.21712225675582886, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 7.858649253845215, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 650.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.3721094131469727, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.545588254928589, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.078303337097168, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.908349990844727, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5206611752510071, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.8985538482666, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.1080283373594284, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.745697975158691, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06868340075016022, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.56973934173584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1694.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.2151605486869812, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.736489295959473, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 20.422527313232422, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.0695855617523193, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.23729589581489563, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.21938052773475647, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 20.0743408203125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 3.1033358573913574, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 42.04832077026367, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.040679678320884705, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.22167710959911346, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20977678894996643, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.02856319397687912, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2496286779642105, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0027213969733566046, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.5151092410087585, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02084008976817131, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.4012624025344849, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 8819.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7547223567962646, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.464192867279053, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.334397077560425, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.100935220718384, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.4186847507953644, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.043750762939453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022521328181028366, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.4528005123138428, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.11789248138666153, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 5.01742696762085, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 18871.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4113915264606476, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.422145366668701, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 25.15517807006836, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 21.08517074584961, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.1487948894500732, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.4664812088012695, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.327824592590332, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.2773406505584717, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 18.006277084350586, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.04044985771179199, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.22361555695533752, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.2094147801399231, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.02305254340171814, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24962158501148224, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.010881972499191761, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.9556485414505005, "metrics/particle/momentum/all/accepted/pz_bias": 0.023630691692233086, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.80303692817688, "metrics/particle/momentum/all/accepted/samples": 13141.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.8258962035179138, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.018802165985107, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.660271406173706, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.437413454055786, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.4291553199291229, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.04071807861328, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.033326346427202225, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.9419193267822266, "metrics/particle/momentum/all/assigned/pz_bias": 0.10890493541955948, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.6244072914123535, "metrics/particle/momentum/all/assigned/samples": 28626.0, "metrics/particle/momentum/all/efficiency": 0.3712776303291321, "metrics/particle/momentum/all/event/energy_bias": -5.651669502258301, "metrics/particle/momentum/all/event/p3_error_mean": 52.48532485961914, "metrics/particle/momentum/all/event/pt_vector_error_mean": 24.897680282592773, "metrics/particle/momentum/all/event/px_bias": 4.1477861404418945, "metrics/particle/momentum/all/event/py_bias": -4.117528438568115, "metrics/particle/momentum/all/event/pz_abs_mean": 39.576377868652344, "metrics/particle/momentum/all/event/pz_bias": 3.721064329147339, "metrics/particle/momentum/all/event/pz_residual_rms": 61.89324188232422, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.049303725361824036, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2452600598335266, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22494564950466156, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.04021531343460083, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.264588862657547, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.015120967291295528, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.1941611766815186, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.022362414747476578, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.691535711288452, "metrics/particle/momentum/class_1/accepted/samples": 5952.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.7875106930732727, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 3.2215576171875, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.90682315826416, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.636784315109253, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.4130757749080658, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.144186019897461, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.04041604697704315, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 2.208620548248291, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.10617908090353012, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 5.872884273529053, "metrics/particle/momentum/class_1/assigned/samples": 13460.0, "metrics/particle/momentum/class_1/efficiency": 0.3566848337650299, "metrics/particle/momentum/class_1/event/energy_bias": 42.12746810913086, "metrics/particle/momentum/class_1/event/p3_error_mean": 37.110130310058594, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 21.618701934814453, "metrics/particle/momentum/class_1/event/px_bias": 4.079782009124756, "metrics/particle/momentum/class_1/event/py_bias": -5.077856540679932, "metrics/particle/momentum/class_1/event/pz_abs_mean": 25.94124412536621, "metrics/particle/momentum/class_1/event/pz_bias": -2.0686020851135254, "metrics/particle/momentum/class_1/event/pz_residual_rms": 36.25044250488281, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04568697512149811, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.25984135270118713, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.24239343404769897, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0881691724061966, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.28096461296081543, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02801724150776863, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7682584524154663, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.16367988288402557, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9698002338409424, "metrics/particle/momentum/class_2/accepted/samples": 928.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.876153290271759, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7876591682434082, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2822329998016357, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8475612998008728, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3773955702781677, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 10.081802368164062, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05552825704216957, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8413920402526855, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.3078574538230896, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.175036430358887, "metrics/particle/momentum/class_2/assigned/samples": 2035.0, "metrics/particle/momentum/class_2/efficiency": 0.3117232024669647, "metrics/particle/momentum/class_2/event/energy_bias": -44.5653076171875, "metrics/particle/momentum/class_2/event/p3_error_mean": 31.293333053588867, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.178832054138184, "metrics/particle/momentum/class_2/event/px_bias": 0.2234797179698944, "metrics/particle/momentum/class_2/event/py_bias": -1.0210210084915161, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.5102481842041, "metrics/particle/momentum/class_2/event/pz_bias": -0.04592818021774292, "metrics/particle/momentum/class_2/event/pz_residual_rms": 36.697330474853516, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030745942145586014, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1956586390733719, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.18799155950546265, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0059944442473351955, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.22736044228076935, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0031265427824109793, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.5643035769462585, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0011697555892169476, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 2.6497550010681152, "metrics/particle/momentum/class_3/accepted/samples": 6077.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8595227599143982, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.354831218719482, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.820748805999756, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.687887191772461, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.45360904932022095, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.780725479125977, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.019893065094947815, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.4589301347732544, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07759083807468414, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.141870975494385, "metrics/particle/momentum/class_3/assigned/samples": 12718.0, "metrics/particle/momentum/class_3/efficiency": 0.39961859583854675, "metrics/particle/momentum/class_3/event/energy_bias": 8.661239624023438, "metrics/particle/momentum/class_3/event/p3_error_mean": 31.51885414123535, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 17.303466796875, "metrics/particle/momentum/class_3/event/px_bias": 0.2638741731643677, "metrics/particle/momentum/class_3/event/py_bias": -0.3736336827278137, "metrics/particle/momentum/class_3/event/pz_abs_mean": 22.63551139831543, "metrics/particle/momentum/class_3/event/pz_bias": 6.7342915534973145, "metrics/particle/momentum/class_3/event/pz_residual_rms": 45.573543548583984, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04749314486980438, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.26962393522262573, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.24599722027778625, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0933728739619255, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2792377769947052, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.051851850003004074, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.3454318046569824, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.23965813219547272, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 4.356243133544922, "metrics/particle/momentum/class_4/accepted/samples": 135.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.8604382276535034, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.199127197265625, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.433321475982666, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.099571704864502, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4542483687400818, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.86943817138672, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.08169934898614883, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 3.3249564170837402, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.13553476333618164, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 7.558544158935547, "metrics/particle/momentum/class_4/assigned/samples": 306.0, "metrics/particle/momentum/class_4/efficiency": 0.3708791136741638, "metrics/particle/momentum/class_4/event/energy_bias": -8.169275283813477, "metrics/particle/momentum/class_4/event/p3_error_mean": 7.000441074371338, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 5.293708324432373, "metrics/particle/momentum/class_4/event/px_bias": -0.07486768066883087, "metrics/particle/momentum/class_4/event/py_bias": 1.1190836429595947, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.645153284072876, "metrics/particle/momentum/class_4/event/pz_bias": 0.015205562114715576, "metrics/particle/momentum/class_4/event/pz_residual_rms": 11.057531356811523, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04986947774887085, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24886879324913025, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.25444281101226807, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.11375189572572708, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.29360154271125793, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.020408162847161293, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2995926141738892, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.2842369079589844, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.374286413192749, "metrics/particle/momentum/class_5/accepted/samples": 49.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.6031365990638733, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7242882251739502, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6622890830039978, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.023137837648391724, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.4579439163208008, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1897693872451782, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.17757008969783783, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.738378047943115, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.31381919980049133, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.657562255859375, "metrics/particle/momentum/class_5/assigned/samples": 107.0, "metrics/particle/momentum/class_5/efficiency": 0.30817610025405884, "metrics/particle/momentum/class_5/event/energy_bias": -3.7057976722717285, "metrics/particle/momentum/class_5/event/p3_error_mean": 7.493206024169922, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 5.993378162384033, "metrics/particle/momentum/class_5/event/px_bias": -0.3444821834564209, "metrics/particle/momentum/class_5/event/py_bias": 1.2358994483947754, "metrics/particle/momentum/class_5/event/pz_abs_mean": 3.6777799129486084, "metrics/particle/momentum/class_5/event/pz_bias": -0.9139022827148438, "metrics/particle/momentum/class_5/event/pz_residual_rms": 14.859540939331055, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.765444278717041, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7714899778366089, "metrics/set_slot/angular_recall_all_dr0p10": 0.7970373630523682, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5013660788536072, "metrics/set_slot/soft_count_bias_mean": 4.628818035125732}, "step_train_time": 6.567808628082275, "step_valid_time": 2841.4986000061035, "step_total_time": 2848.066409111023, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0955412076120297, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.36999382786430024, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2916, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2276, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1969, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7518995705318797, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7805212620027435, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7659431263671547, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2194787379972565, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6504790221341262, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6752400548696845, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6626283022042739, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7518995705318797, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099545940938892, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.36372442024901064, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2110, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1686, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1546, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7453580901856764, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7990521327014218, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7712717291857274, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2009478672985782, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6834659593280283, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7327014218009479, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.707227813357731, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7453580901856764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9067887935781767, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3927784452875681, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1152, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 876, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 777, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.6801242236024845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7604166666666666, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7180327868852459, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.23958333333333334, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6032608695652174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6744791666666666, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6368852459016393, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.6801242236024845} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_10000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..caf8a83f080a7f7e9c386e5854f8ea5e05a191de --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47375402690922874, "train": {"Classification_binary": 0.3104247748851776, "Cardinality": 0.19326503574848175, "Classification": 0.16061089932918549, "Regression_pt": 0.23869836330413818, "Regression_eta": 0.0036072167567908764, "Regression_sin_phi": 0.0019515580497682095, "Regression_cos_phi": 0.001968683674931526, "Regression_energy": 0.23528103530406952, "HitGrouping": 0.19383607804775238, "Auxiliary": 0.32657331228256226, "Total": 1.765146017074585, "Total_unweighted": 0.9525425313040614}, "train_group": {}, "valid": {"Classification_binary": 0.25769855578740436, "Cardinality": 0.17068896691004434, "Classification": 0.1437826951344808, "Regression_pt": 0.23883944749832153, "Regression_eta": 0.0032211318612098694, "Regression_sin_phi": 0.0017781673620144527, "Regression_cos_phi": 0.0017135182085136573, "Regression_energy": 0.23717892169952393, "HitGrouping": 0.19612022240956625, "Auxiliary": 0.301180362701416, "Total": 1.614871342976888, "Total_unweighted": 0.8842124375514686, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8262457847595215, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4307534992694855, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36780956387519836, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.43348896503448486, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4091554880142212, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41069236397743225, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7790544033050537, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4345904588699341, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3727549910545349, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.43751323223114014, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.40305808186531067, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4828871488571167, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8363920450210571, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4241693913936615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3324553370475769, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4238969683647156, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.40189802646636963, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.39262473583221436, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8279841542243958, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4272228181362152, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3474467992782593, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.42820489406585693, "metrics/hit_grouping/calorimeter/all/object_purity": 0.40433749556541443, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40653494000434875, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025411490350961685, "metrics/hit_grouping/calorimeter/background_recall": 0.6268190741539001, "metrics/hit_grouping/calorimeter/completeness": 0.5126811265945435, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5292430520057678, "metrics/hit_grouping/calorimeter/energy_purity": 0.5230136513710022, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4387478530406952, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.848723828792572, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.441092848777771, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.284920871257782, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.44519689679145813, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4207235276699066, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.35759928822517395, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8243021965026855, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.44711703062057495, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3683891296386719, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4461970031261444, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4274289608001709, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.419335275888443, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.787921667098999, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.37713301181793213, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45796990394592285, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.37348905205726624, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.34559008479118347, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49677664041519165, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8076575994491577, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4459126591682434, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.409342885017395, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4493301212787628, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.43225857615470886, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.45400187373161316, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8115520477294922, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5818140506744385, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5604395866394043, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.576309323310852, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5172659754753113, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.41496598720550537, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8519390821456909, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.38734808564186096, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25364938378334045, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3865421712398529, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.35986989736557007, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35442614555358887, "metrics/hit_grouping/calorimeter/purity": 0.5049343705177307, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8898101449012756, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0714210718870163, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12153846025466919, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0721684917807579, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.028748909011483192, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04153846204280853, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.843482494354248, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1775745004415512, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11117445677518845, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.17616523802280426, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1579241007566452, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2896237075328827, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8257749080657959, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4491651952266693, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3658512532711029, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.45028984546661377, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.42651551961898804, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.42103883624076843, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7007251977920532, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4478481709957123, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.40905097126960754, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.40276479721069336, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6439031958580017, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7798753976821899, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.37969714403152466, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46617692708969116, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.44236835837364197, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4784071743488312, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.637268602848053, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4508151412010193, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3368174135684967, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.33879658579826355, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7272583842277527, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6693354845046997, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4438881576061249, "metrics/hit_grouping/tracker/all/object_completeness": 0.37054288387298584, "metrics/hit_grouping/tracker/all/object_purity": 0.3677378296852112, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6799546480178833, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0056457542814314365, "metrics/hit_grouping/tracker/background_recall": 0.04211336746811867, "metrics/hit_grouping/tracker/completeness": 0.3891707956790924, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5499555468559265, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.689677357673645, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.44346240162849426, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4455172121524811, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4381808042526245, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6573120951652527, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6432294249534607, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.446601927280426, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.32567131519317627, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3394785523414612, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7211117744445801, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6519169211387634, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44194984436035156, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.25178617238998413, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.24138961732387543, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.686488687992096, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.676490843296051, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45920202136039734, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3677087426185608, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3730505704879761, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6696621179580688, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7023805379867554, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5159786939620972, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3984350264072418, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.388094037771225, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6304926872253418, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6542032957077026, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.41126200556755066, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3693305253982544, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3572373390197754, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7020604014396667, "metrics/hit_grouping/tracker/purity": 0.37122631072998047, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8830220699310303, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.021720243617892265, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.06433258950710297, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0725601464509964, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0842745453119278, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6571986079216003, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20995333790779114, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1939050555229187, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.19280490279197693, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6127527356147766, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6416809558868408, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5278909206390381, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4322647154331207, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.42776644229888916, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7685496807098389, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.49060943722724915, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.47448834776878357, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.514199435710907, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5944840312004089, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0048820823431015015, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06367926299571991, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.428207397460938, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.24179039895534515, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.11135056614875793, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7116343379020691, "metrics/particle/class_1/pid_accuracy": 0.941917359828949, "metrics/particle/class_2/efficiency": 0.40509310364723206, "metrics/particle/class_2/pid_accuracy": 0.5491553544998169, "metrics/particle/class_3/efficiency": 0.581061065196991, "metrics/particle/class_3/pid_accuracy": 0.9333209991455078, "metrics/particle/class_4/efficiency": 0.6073147058486938, "metrics/particle/class_4/pid_accuracy": 0.31378763914108276, "metrics/particle/class_5/efficiency": 0.7179999947547913, "metrics/particle/class_5/pid_accuracy": 0.42339831590652466, "metrics/particle/count/bias_mean": -8.391926765441895, "metrics/particle/count/mae": 8.940104484558105, "metrics/particle/count/prediction_mean": 61.6640625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09851331263780594, "metrics/particle/event/energy_response_mean": 1.0063217878341675, "metrics/particle/event/met_abs_error": 11.656578063964844, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08705409616231918, "metrics/particle/event/scalar_pt_response_mean": 1.0167876482009888, "metrics/particle/event/vector_pt_closure": 0.0865899920463562, "metrics/particle/matched/delta_eta_abs_mean": 0.01905359886586666, "metrics/particle/matched/delta_phi_abs_mean": 0.0185637716203928, "metrics/particle/matched/delta_r_mean": 0.029797472059726715, "metrics/particle/matched/energy_relative_abs_mean": 0.17527703940868378, "metrics/particle/matched/pid_accuracy": 0.9089014530181885, "metrics/particle/matched/pt_relative_abs_mean": 0.1720060557126999, "metrics/particle/matching/duplicate_fraction": 0.09189577400684357, "metrics/particle/matching/efficiency": 0.6287010312080383, "metrics/particle/matching/f1": 0.6687557697296143, "metrics/particle/matching/purity": 0.7142615914344788, "metrics/particle/matching/target_coverage_dr0p05": 0.5566232204437256, "metrics/particle/matching/target_coverage_dr0p10": 0.6944594383239746, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4257476329803467, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6855379939079285, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0283490177243948, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1721525937318802, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16717363893985748, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.006073945667594671, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20669278502464294, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01096651516854763, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9717999696731567, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08643700927495956, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3717029094696045, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6839.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.34533798694610596, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7573553323745728, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7777559757232666, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.594123363494873, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.19892413914203644, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.942110061645508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.024481281638145447, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5042170286178589, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.1681983470916748, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.118780136108398, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9109.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6253657937049866, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8301361799240112, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.623064994812012, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.764732837677002, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.569543719291687, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.588019847869873, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.825374603271484, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.191859245300293, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.034832000732422, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026511913165450096, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17458392679691315, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16959726810455322, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0012826272286474705, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2090742588043213, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02636309154331684, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7077637910842896, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.23370468616485596, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.165661811828613, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1669.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.364808589220047, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7503305077552795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.6043798923492432, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.4291329383850098, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17829099297523499, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 9.195613861083984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05542725324630737, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.505587100982666, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.19526727497577667, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.048300743103027, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2165.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.552466094493866, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.260293483734131, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.119455337524414, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.815398931503296, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07033950090408325, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11225465685129166, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.911069869995117, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4374145269393921, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.464494705200195, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03034418262541294, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1739458441734314, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17226262390613556, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.009562446735799313, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21239207684993744, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001162283937446773, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.33718326687812805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.023581212386488914, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9045448303222656, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13766.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.38460758328437805, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.584549903869629, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.951616644859314, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7570183277130127, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.23271235823631287, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.052350997924805, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0095676826313138, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7690460085868835, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04716729000210762, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1726343631744385, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19754.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6421607732772827, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.939175605773926, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.854263305664062, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.629521369934082, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.8322097063064575, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.0673065185546875, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.123958587646484, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.379995346069336, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.135397911071777, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.029444433748722076, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17344306409358978, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17050038278102875, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007870927453041077, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2104097306728363, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0060608782805502415, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6347339153289795, "metrics/particle/momentum/all/accepted/pz_bias": 0.05862507224082947, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.879630446434021, "metrics/particle/momentum/all/accepted/samples": 22274.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3716976046562195, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.2137231826782227, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.8763470649719238, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.686318278312683, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.21899574995040894, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.638510704040527, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017145803198218346, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1060410737991333, "metrics/particle/momentum/all/assigned/pz_bias": 0.0930325910449028, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.7337772846221924, "metrics/particle/momentum/all/assigned/samples": 31028.0, "metrics/particle/momentum/all/efficiency": 0.6293156743049622, "metrics/particle/momentum/all/event/energy_bias": 2.5090181827545166, "metrics/particle/momentum/all/event/p3_error_mean": 26.75233268737793, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.286920547485352, "metrics/particle/momentum/all/event/px_bias": 2.472093343734741, "metrics/particle/momentum/all/event/py_bias": 3.767580986022949, "metrics/particle/momentum/all/event/pz_abs_mean": 19.119033813476562, "metrics/particle/momentum/all/event/pz_bias": 5.0092692375183105, "metrics/particle/momentum/all/event/pz_residual_rms": 29.6669864654541, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.034829478710889816, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17694906890392303, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17286431789398193, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.021925970911979675, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21198202669620514, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006121079903095961, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6874136924743652, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.09829211235046387, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8574100732803345, "metrics/particle/momentum/class_1/accepted/samples": 11926.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.25660452246665955, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8501105308532715, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.8268231153488159, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.6180716156959534, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1611623913049698, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.141176700592041, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01339227706193924, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.98621666431427, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.1463000774383545, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.08571195602417, "metrics/particle/momentum/class_1/assigned/samples": 15382.0, "metrics/particle/momentum/class_1/efficiency": 0.7146880626678467, "metrics/particle/momentum/class_1/event/energy_bias": 11.360379219055176, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.060285568237305, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.69286823272705, "metrics/particle/momentum/class_1/event/px_bias": 0.31576821208000183, "metrics/particle/momentum/class_1/event/py_bias": 3.955641984939575, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.388059616088867, "metrics/particle/momentum/class_1/event/pz_bias": 7.299349784851074, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.547353744506836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03518296033143997, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20914022624492645, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2098362147808075, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0009833460208028555, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24912995100021362, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02976190485060215, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7807745933532715, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09195106476545334, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9579858779907227, "metrics/particle/momentum/class_2/accepted/samples": 1176.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4883742034435272, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7724224328994751, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9244252443313599, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6371356844902039, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3044140040874481, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.574109077453613, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05327245220541954, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5913429260253906, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.036356329917907715, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.911372661590576, "metrics/particle/momentum/class_2/assigned/samples": 1971.0, "metrics/particle/momentum/class_2/efficiency": 0.39502856135368347, "metrics/particle/momentum/class_2/event/energy_bias": -5.619035243988037, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.073322296142578, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.868695259094238, "metrics/particle/momentum/class_2/event/px_bias": 2.726900339126587, "metrics/particle/momentum/class_2/event/py_bias": -0.649401843547821, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.63802433013916, "metrics/particle/momentum/class_2/event/pz_bias": -3.5840299129486084, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.436702728271484, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021231211721897125, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16363435983657837, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16192050278186798, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010096734389662743, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20225849747657776, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0023798730690032244, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3817175626754761, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01156550645828247, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3521919250488281, "metrics/particle/momentum/class_3/accepted/samples": 8824.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4907647669315338, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.012975692749023, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.2418160438537598, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0882651805877686, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2734919786453247, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.923925399780273, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01515610795468092, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9933754205703735, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029673436656594276, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.963595390319824, "metrics/particle/momentum/class_3/assigned/samples": 13196.0, "metrics/particle/momentum/class_3/efficiency": 0.5802590847015381, "metrics/particle/momentum/class_3/event/energy_bias": -0.9108583927154541, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.678863525390625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.959794998168945, "metrics/particle/momentum/class_3/event/px_bias": -0.3626108765602112, "metrics/particle/momentum/class_3/event/py_bias": 0.13347119092941284, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.156805038452148, "metrics/particle/momentum/class_3/event/pz_bias": 0.3684953451156616, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.541269302368164, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03432317450642586, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17027626931667328, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1599639356136322, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0008406435372307897, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2023146003484726, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.012987012974917889, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1202298402786255, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4894583225250244, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.67842960357666, "metrics/particle/momentum/class_4/accepted/samples": 231.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40219780802726746, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.202197551727295, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4878036975860596, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3346824645996094, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23475609719753265, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.726720809936523, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.021341463550925255, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3869096040725708, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5468435287475586, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.816622257232666, "metrics/particle/momentum/class_4/assigned/samples": 328.0, "metrics/particle/momentum/class_4/efficiency": 0.6346153616905212, "metrics/particle/momentum/class_4/event/energy_bias": -1.5534064769744873, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.25068998336792, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1399221420288086, "metrics/particle/momentum/class_4/event/px_bias": 0.10360625386238098, "metrics/particle/momentum/class_4/event/py_bias": 0.1652974784374237, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.080291748046875, "metrics/particle/momentum/class_4/event/pz_bias": 0.7368770241737366, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.241999626159668, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03265829011797905, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20327794551849365, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2020527720451355, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03651390224695206, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24086767435073853, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025641025975346565, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8694730997085571, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.12340052425861359, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3790555000305176, "metrics/particle/momentum/class_5/accepted/samples": 117.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10135073214769363, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5498412251472473, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5566518902778625, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2754165828227997, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1986754983663559, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.970536947250366, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09271523356437683, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.160492181777954, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04216139763593674, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.833937644958496, "metrics/particle/momentum/class_5/assigned/samples": 151.0, "metrics/particle/momentum/class_5/efficiency": 0.7358490824699402, "metrics/particle/momentum/class_5/event/energy_bias": -0.7680609226226807, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.634124994277954, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.7207748889923096, "metrics/particle/momentum/class_5/event/px_bias": -0.31157079339027405, "metrics/particle/momentum/class_5/event/py_bias": 0.16257193684577942, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.8928413391113281, "metrics/particle/momentum/class_5/event/pz_bias": 0.18857648968696594, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.045906066894531, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.531899631023407, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5334358811378479, "metrics/set_slot/angular_recall_all_dr0p10": 0.8726744055747986, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7050815224647522, "metrics/set_slot/soft_count_bias_mean": -1.1102365255355835}, "step_train_time": 6.449670314788818, "step_valid_time": 31028.04343008995, "step_total_time": 31034.493101358414, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.042633312368542, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21978345868689142, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2774, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2655, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.916418896597291, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9176314918954681, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9170247933884298, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08236850810453192, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8771060455896927, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8782666225603705, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8776859504132232, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.916418896597291, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005234522115309, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21055944254266168, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2068, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2039, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9142351900972591, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9228023203926818, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9184987785920498, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07719767960731816, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9014146772767463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9098616688978135, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9056184765711748, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9142351900972591, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0106392541528022, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22735101353003018, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1262, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1154, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1122, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8959627329192547, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9144215530903328, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9050980392156863, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08557844690966719, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8711180124223602, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8890649762282092, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.88, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8959627329192547} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_11000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..2887a8d7e2035b7889e7625d59baf42054c509bb --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5211294296001516, "train": {"Classification_binary": 0.294454425573349, "Cardinality": 0.2080046832561493, "Classification": 0.14862507581710815, "Regression_pt": 0.21488706767559052, "Regression_eta": 0.0030904465820640326, "Regression_sin_phi": 0.0016649499302729964, "Regression_cos_phi": 0.001621572533622384, "Regression_energy": 0.21031929552555084, "HitGrouping": 0.1877676248550415, "Auxiliary": 0.3098772466182709, "Total": 1.6462173461914062, "Total_unweighted": 0.8746628336375579}, "train_group": {}, "valid": {"Classification_binary": 0.24655936161677042, "Cardinality": 0.16018658876419067, "Classification": 0.1271066665649414, "Regression_pt": 0.20426803827285767, "Regression_eta": 0.002699844539165497, "Regression_sin_phi": 0.0015337431492904823, "Regression_cos_phi": 0.0014783370619018872, "Regression_energy": 0.20118331909179688, "HitGrouping": 0.1881932814915975, "Auxiliary": 0.2777525583902995, "Total": 1.4640474319458008, "Total_unweighted": 0.7848293102967242, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8256067037582397, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.46759939193725586, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35562482476234436, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4713480770587921, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.45458224415779114, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4028525650501251, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7748790383338928, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4624927341938019, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.36258894205093384, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4671316146850586, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.44954925775527954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.47475433349609375, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.841651976108551, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4553525149822235, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3346245288848877, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.45682305097579956, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.43486058712005615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3704162836074829, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8304994702339172, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.45993393659591675, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34389057755470276, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4624216556549072, "metrics/hit_grouping/calorimeter/all/object_purity": 0.44252440333366394, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3902127742767334, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02564944513142109, "metrics/hit_grouping/calorimeter/background_recall": 0.627170741558075, "metrics/hit_grouping/calorimeter/completeness": 0.5396884679794312, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5534218549728394, "metrics/hit_grouping/calorimeter/energy_purity": 0.5465763807296753, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4226565361022949, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8521019816398621, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.47238385677337646, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2718423306941986, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4778051972389221, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.45559635758399963, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.33610033988952637, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8288981914520264, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4845321774482727, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36559921503067017, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4860735535621643, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4712369441986084, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.40065503120422363, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7864429354667664, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.40794774889945984, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47377070784568787, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.40521240234375, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.38493409752845764, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4938693046569824, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8106461763381958, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4785592555999756, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4067891538143158, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4839038848876953, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.47196367383003235, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44067269563674927, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8021630048751831, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5913821458816528, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5478807091712952, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5878201127052307, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5524093508720398, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4285714328289032, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8554689884185791, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4230899214744568, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2501673996448517, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.42328065633773804, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3968140482902527, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3310566544532776, "metrics/hit_grouping/calorimeter/purity": 0.5316633582115173, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8795737624168396, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06010790914297104, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10923077166080475, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06083516776561737, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03161082789301872, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03384615480899811, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.836519718170166, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2216348797082901, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.13169898092746735, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22184179723262787, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20461711287498474, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.25370582938194275, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8291072845458984, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4821617901325226, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3610965311527252, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.48481839895248413, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4649661183357239, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4056597650051117, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6916418075561523, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4481470286846161, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4331282675266266, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.46395573019981384, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6546623110771179, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7720347046852112, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3853056728839874, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5075153112411499, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4934832453727722, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48681995272636414, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.635016679763794, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45278412103652954, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.35554301738739014, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.37297651171684265, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7235567569732666, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6644536256790161, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44563573598861694, "metrics/hit_grouping/tracker/all/object_completeness": 0.3928643465042114, "metrics/hit_grouping/tracker/all/object_purity": 0.4118815064430237, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6818439364433289, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0037546856328845024, "metrics/hit_grouping/tracker/background_recall": 0.03453177586197853, "metrics/hit_grouping/tracker/completeness": 0.4116581976413727, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.519381582736969, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6936506032943726, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4316173195838928, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46888071298599243, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4803132116794586, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6452847123146057, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6343834400177002, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45954692363739014, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3497355580329895, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3898407220840454, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7310108542442322, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6315898895263672, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4619741141796112, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.26994258165359497, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.28339096903800964, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7107605338096619, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6689844131469727, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46935656666755676, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3928368389606476, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.42081016302108765, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6769410371780396, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.67920982837677, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46804261207580566, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.393289178609848, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4325498938560486, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.666444718837738, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6559564471244812, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4107937216758728, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3928254246711731, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.396615594625473, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6909388899803162, "metrics/hit_grouping/tracker/purity": 0.3919314444065094, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.882569432258606, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.026498697698116302, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.07656010240316391, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.08476496487855911, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08210252225399017, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6494307518005371, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24261274933815002, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22957377135753632, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2456488460302353, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6168999671936035, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6365258693695068, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5257068872451782, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4544380307197571, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.475259393453598, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7707337141036987, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5147423148155212, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5060133337974548, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5532569289207458, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5877394676208496, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.029014967381954193, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05693471059203148, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.959016799926758, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2512974441051483, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.12085762619972229, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.755955696105957, "metrics/particle/class_1/pid_accuracy": 0.94749516248703, "metrics/particle/class_2/efficiency": 0.43740183115005493, "metrics/particle/class_2/pid_accuracy": 0.5783534049987793, "metrics/particle/class_3/efficiency": 0.6124257445335388, "metrics/particle/class_3/pid_accuracy": 0.9426559805870056, "metrics/particle/class_4/efficiency": 0.6102021336555481, "metrics/particle/class_4/pid_accuracy": 0.3927444815635681, "metrics/particle/class_5/efficiency": 0.7940000295639038, "metrics/particle/class_5/pid_accuracy": 0.45843827724456787, "metrics/particle/count/bias_mean": -5.64453125, "metrics/particle/count/mae": 6.658854007720947, "metrics/particle/count/prediction_mean": 64.41146087646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09235471487045288, "metrics/particle/event/energy_response_mean": 0.9521264433860779, "metrics/particle/event/met_abs_error": 9.34170913696289, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07637981325387955, "metrics/particle/event/scalar_pt_response_mean": 0.9763770699501038, "metrics/particle/event/vector_pt_closure": 0.07346971333026886, "metrics/particle/matched/delta_eta_abs_mean": 0.01703784242272377, "metrics/particle/matched/delta_phi_abs_mean": 0.01763150468468666, "metrics/particle/matched/delta_r_mean": 0.02750377543270588, "metrics/particle/matched/energy_relative_abs_mean": 0.16407053172588348, "metrics/particle/matched/pid_accuracy": 0.9178745150566101, "metrics/particle/matched/pt_relative_abs_mean": 0.16378365457057953, "metrics/particle/matching/duplicate_fraction": 0.0886835902929306, "metrics/particle/matching/efficiency": 0.6661617159843445, "metrics/particle/matching/f1": 0.6941251754760742, "metrics/particle/matching/purity": 0.7245391011238098, "metrics/particle/matching/target_coverage_dr0p05": 0.6019366979598999, "metrics/particle/matching/target_coverage_dr0p10": 0.7289835214614868, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4707079529762268, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7249038219451904, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026166364550590515, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1593739241361618, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15940527617931366, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010055034421384335, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19964873790740967, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010482758283615112, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8926277756690979, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03283747658133507, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3717684745788574, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7250.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3607467710971832, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2478536367416382, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.453395128250122, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2804540395736694, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1944127082824707, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.57784652709961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01973205991089344, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2858701944351196, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12123525142669678, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.347473621368408, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9629.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6629480719566345, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.260728120803833, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.009665489196777, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.10241174697876, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1702163815498352, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0867338627576828, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.234370231628418, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.405709147453308, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.888833045959473, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025117792189121246, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16955557465553284, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16968092322349548, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026525253430008888, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2102181762456894, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0377030149102211, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.759377360343933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0012458217097446322, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.269121170043945, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1724.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.48025137186050415, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9360331296920776, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.361987590789795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2025606632232666, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20242740213871002, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.406087875366211, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05765062943100929, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.414763927459717, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.015066318213939667, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.563207149505615, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2307.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5706719756126404, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.147872924804688, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.87863826751709, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6057820320129395, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.19067254662513733, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05369279161095619, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.70328140258789, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.1052379310131073, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.11951446533203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028331462293863297, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1648263931274414, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16473907232284546, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0009955578716471791, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2051960974931717, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010236112866550684, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3117406666278839, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009001332335174084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8817713260650635, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14654.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3591025769710541, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.8236147165298462, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3776466846466064, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1867679357528687, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.20761142671108246, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.224504470825195, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007404116913676262, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6478049755096436, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04317106306552887, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.6213741302490234, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20259.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.683584451675415, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.3233509063720703, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.911076545715332, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.829845428466797, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9283521771430969, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.29018115997314453, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.539604663848877, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.9161062240600586, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.075258255004883, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027432642877101898, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1634984165430069, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1634630560874939, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004403234459459782, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20388367772102356, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006602336186915636, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5956054329872131, "metrics/particle/momentum/all/accepted/pz_bias": 0.00440233713015914, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.880977749824524, "metrics/particle/momentum/all/accepted/samples": 23628.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3682754635810852, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5878125429153442, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4708367586135864, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.287576675415039, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.20329242944717407, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.679080963134766, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014691722579300404, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.965255081653595, "metrics/particle/momentum/all/assigned/pz_bias": 0.06234566867351532, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1453394889831543, "metrics/particle/momentum/all/assigned/samples": 32195.0, "metrics/particle/momentum/all/efficiency": 0.6675707697868347, "metrics/particle/momentum/all/event/energy_bias": -12.731950759887695, "metrics/particle/momentum/all/event/p3_error_mean": 24.91312599182129, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.095866203308105, "metrics/particle/momentum/all/event/px_bias": -1.289241075515747, "metrics/particle/momentum/all/event/py_bias": -0.4306078553199768, "metrics/particle/momentum/all/event/pz_abs_mean": 18.62099838256836, "metrics/particle/momentum/all/event/pz_bias": 3.4270529747009277, "metrics/particle/momentum/all/event/pz_residual_rms": 30.102615356445312, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03141383081674576, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15983431041240692, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.160077765583992, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.013066906481981277, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19956916570663452, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006603254470974207, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6328500509262085, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.023524900898337364, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8088788986206055, "metrics/particle/momentum/class_1/accepted/samples": 12721.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2386426478624344, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6186352968215942, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6348502039909363, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.43908482789993286, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1380358338356018, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.22720193862915, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011650731787085533, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8542509078979492, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.07292996346950531, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6129016876220703, "metrics/particle/momentum/class_1/assigned/samples": 15793.0, "metrics/particle/momentum/class_1/efficiency": 0.7623299360275269, "metrics/particle/momentum/class_1/event/energy_bias": 5.804963111877441, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.16958236694336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.594220161437988, "metrics/particle/momentum/class_1/event/px_bias": -0.3389987647533417, "metrics/particle/momentum/class_1/event/py_bias": 0.9061304926872253, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.09954833984375, "metrics/particle/momentum/class_1/event/pz_bias": 3.1661999225616455, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.036836624145508, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034161899238824844, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2073862999677658, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20499835908412933, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03941578045487404, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24627268314361572, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03164556995034218, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7624499797821045, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07869420200586319, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.237009048461914, "metrics/particle/momentum/class_2/accepted/samples": 1264.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5824759602546692, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7340519428253174, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8735203146934509, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5004850625991821, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28870889544487, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.728697776794434, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05145307257771492, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4330923557281494, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.12448251247406006, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.514561653137207, "metrics/particle/momentum/class_2/assigned/samples": 2099.0, "metrics/particle/momentum/class_2/efficiency": 0.42458850145339966, "metrics/particle/momentum/class_2/event/energy_bias": -18.04256820678711, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.147661209106445, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.297420501708984, "metrics/particle/momentum/class_2/event/px_bias": 0.18473683297634125, "metrics/particle/momentum/class_2/event/py_bias": -0.8487318754196167, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.466157913208008, "metrics/particle/momentum/class_2/event/pz_bias": 0.5021336078643799, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.65697479248047, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02103430964052677, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16205818951129913, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16191139817237854, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03405456244945526, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20272241532802582, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002474981127306819, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3521334230899811, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02947041764855385, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.297333002090454, "metrics/particle/momentum/class_3/accepted/samples": 9293.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.48636874556541443, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.824732780456543, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.516345977783203, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.37626314163208, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2643079459667206, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.906426429748535, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011866001412272453, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8408671617507935, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03870278224349022, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.169353485107422, "metrics/particle/momentum/class_3/assigned/samples": 13821.0, "metrics/particle/momentum/class_3/efficiency": 0.6111001372337341, "metrics/particle/momentum/class_3/event/energy_bias": 0.8165870904922485, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.41018009185791, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.171266555786133, "metrics/particle/momentum/class_3/event/px_bias": -1.1044968366622925, "metrics/particle/momentum/class_3/event/py_bias": -0.5097113847732544, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.791909217834473, "metrics/particle/momentum/class_3/event/pz_bias": -0.9355581998825073, "metrics/particle/momentum/class_3/event/pz_residual_rms": 21.010698318481445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029107488691806793, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16285674273967743, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16226136684417725, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05294782295823097, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20498071610927582, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008771929889917374, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0771582126617432, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2810927629470825, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.609138011932373, "metrics/particle/momentum/class_4/accepted/samples": 228.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3954930603504181, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.111013174057007, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9592316150665283, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8296053409576416, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2507462799549103, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.4539079666137695, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011940298601984978, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2705557346343994, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.372978150844574, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.539606809616089, "metrics/particle/momentum/class_4/assigned/samples": 335.0, "metrics/particle/momentum/class_4/efficiency": 0.6263736486434937, "metrics/particle/momentum/class_4/event/energy_bias": 0.09557995200157166, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.093799114227295, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.756737470626831, "metrics/particle/momentum/class_4/event/px_bias": 0.1978849470615387, "metrics/particle/momentum/class_4/event/py_bias": -0.19295431673526764, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.5264129638671875, "metrics/particle/momentum/class_4/event/pz_bias": 0.8768101930618286, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.8453240394592285, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.026837190613150597, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20175768435001373, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20655207335948944, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06931041181087494, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24596989154815674, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05737704783678055, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.268655300140381, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6961628198623657, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.1384382247924805, "metrics/particle/momentum/class_5/accepted/samples": 122.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07167565822601318, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4143980145454407, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4021952748298645, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0904560536146164, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14965985715389252, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.3126490116119385, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08843537420034409, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.931154727935791, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.44701820611953735, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.104145526885986, "metrics/particle/momentum/class_5/assigned/samples": 147.0, "metrics/particle/momentum/class_5/efficiency": 0.7672955989837646, "metrics/particle/momentum/class_5/event/energy_bias": -1.4065132141113281, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.104412078857422, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.3777542114257812, "metrics/particle/momentum/class_5/event/px_bias": -0.2283673882484436, "metrics/particle/momentum/class_5/event/py_bias": 0.21465922892093658, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6127257347106934, "metrics/particle/momentum/class_5/event/pz_bias": -0.1825323849916458, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.940269947052002, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4794773459434509, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.48221254348754883, "metrics/set_slot/angular_recall_all_dr0p10": 0.8915674090385437, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7303124070167542, "metrics/set_slot/soft_count_bias_mean": 1.1472986936569214}, "step_train_time": 5.107780456542969, "step_valid_time": 34275.56118798256, "step_total_time": 34280.66896867752, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0116250696516589, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19661105397541956, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2813, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2762, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9293029402048233, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9311486262826878, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9302248677248677, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06885137371731215, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9124545754872811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9142667990731546, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9133597883597884, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9293029402048233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9890404918836767, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19386367979903507, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2102, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2088, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9292661361626879, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.937137761925992, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9331853496115428, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06286223807400802, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9230769230769231, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9308961212661614, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9269700332963374, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9292661361626879, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9909813966478777, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18275391566637245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1246, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1147, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1134, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.890527950310559, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.920545746388443, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9052880820836622, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07945425361155699, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8804347826086957, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9101123595505618, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8950276243093923, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.890527950310559} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_12000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..20e9a63b096dc0cf859143dd294c22b610e5fa06 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5685048322910745, "train": {"Classification_binary": 0.27573564648628235, "Cardinality": 0.17695271968841553, "Classification": 0.13258469104766846, "Regression_pt": 0.20865337550640106, "Regression_eta": 0.00268533849157393, "Regression_sin_phi": 0.001553547685034573, "Regression_cos_phi": 0.0014505416620522738, "Regression_energy": 0.2041982263326645, "HitGrouping": 0.18628202378749847, "Auxiliary": 0.295320987701416, "Total": 1.5317579507827759, "Total_unweighted": 0.8268613672116771}, "train_group": {}, "valid": {"Classification_binary": 0.23630332946777344, "Cardinality": 0.14814001321792603, "Classification": 0.11685237288475037, "Regression_pt": 0.19342106580734253, "Regression_eta": 0.0024546366184949875, "Regression_sin_phi": 0.0014180618648727734, "Regression_cos_phi": 0.001388498271505038, "Regression_energy": 0.19100522994995117, "HitGrouping": 0.18407529592514038, "Auxiliary": 0.2620621124903361, "Total": 1.3824647267659504, "Total_unweighted": 0.7428431948646903, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8277291655540466, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4788844585418701, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3403230309486389, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.48222386837005615, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.46786028146743774, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.39841315150260925, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7776249647140503, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4603782892227173, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3388681709766388, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.46416908502578735, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.460299015045166, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.47407659888267517, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8399222493171692, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4722042977809906, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3335399329662323, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.473268061876297, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4551111161708832, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3743414878845215, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8304107785224915, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4732931852340698, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3362005949020386, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4753338396549225, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4596790373325348, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39103344082832336, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025998327881097794, "metrics/hit_grouping/calorimeter/background_recall": 0.6360046863555908, "metrics/hit_grouping/calorimeter/completeness": 0.5472785234451294, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5575042963027954, "metrics/hit_grouping/calorimeter/energy_purity": 0.5509718656539917, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4091532826423645, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8510687947273254, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.48677897453308105, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2693938612937927, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4920395314693451, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.47419920563697815, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.33484622836112976, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8324859142303467, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4968714118003845, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.35383307933807373, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.49813738465309143, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.48397207260131836, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4017467200756073, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7845221757888794, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4201774299144745, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45923396944999695, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4162101149559021, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.40362903475761414, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4987991452217102, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8110227584838867, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4920523166656494, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3968234062194824, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4970550835132599, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.49262380599975586, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44185611605644226, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8106898069381714, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5955149531364441, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5306122303009033, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5895152688026428, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5553390383720398, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.41810569167137146, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8537775278091431, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.43748605251312256, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2461497187614441, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.437371164560318, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4120204448699951, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3329315781593323, "metrics/hit_grouping/calorimeter/purity": 0.539333701133728, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8766922950744629, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08616305142641068, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1261538416147232, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08784615248441696, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04285305365920067, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.829491138458252, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.239360049366951, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1214367151260376, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2367992401123047, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21365782618522644, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.258266806602478, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8294771909713745, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4949994385242462, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.35302990674972534, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.49731236696243286, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4827134609222412, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.40621721744537354, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6996562480926514, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4324566721916199, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4415168762207031, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4699230492115021, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6446503400802612, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7848628759384155, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34772855043411255, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5055423974990845, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5007298588752747, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45709478855133057, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6283867359161377, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45900607109069824, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3649604022502899, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3965550661087036, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7367882132530212, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6640911102294922, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44124314188957214, "metrics/hit_grouping/tracker/all/object_completeness": 0.4009973108768463, "metrics/hit_grouping/tracker/all/object_purity": 0.428518146276474, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6841111183166504, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00391329126432538, "metrics/hit_grouping/tracker/background_recall": 0.024225551635026932, "metrics/hit_grouping/tracker/completeness": 0.4208776354789734, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5091479420661926, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.695042073726654, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42642369866371155, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47636112570762634, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4952879846096039, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6492027044296265, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6354157328605652, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45174187421798706, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.35768163204193115, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.404734343290329, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7308204770088196, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6258517503738403, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.46298542618751526, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.27972298860549927, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.30556854605674744, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7119740843772888, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6609244346618652, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46279653906822205, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.39874544739723206, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.44316840171813965, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6868260502815247, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6809200048446655, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4427430033683777, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.39358142018318176, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.436877578496933, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6544607281684875, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6652573943138123, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4129009544849396, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4052349030971527, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40796270966529846, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6857878565788269, "metrics/hit_grouping/tracker/purity": 0.4007728695869446, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8876727819442749, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02823631651699543, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.07303214073181152, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.07929639518260956, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08601216226816177, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6557024717330933, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24313116073608398, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.23786461353302002, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2478470802307129, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5966821908950806, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.634665310382843, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5199220776557922, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.46413761377334595, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4965437948703766, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7753379344940186, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4967060685157776, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5189902782440186, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5693137049674988, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5764299035072327, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.010978737846016884, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04562509432435036, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.860530853271484, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26180973649024963, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.13136990368366241, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7721210718154907, "metrics/particle/class_1/pid_accuracy": 0.953873336315155, "metrics/particle/class_2/efficiency": 0.4077855050563812, "metrics/particle/class_2/pid_accuracy": 0.6033012270927429, "metrics/particle/class_3/efficiency": 0.6503121256828308, "metrics/particle/class_3/pid_accuracy": 0.9441164135932922, "metrics/particle/class_4/efficiency": 0.6419634222984314, "metrics/particle/class_4/pid_accuracy": 0.35082459449768066, "metrics/particle/class_5/efficiency": 0.8500000238418579, "metrics/particle/class_5/pid_accuracy": 0.4870588183403015, "metrics/particle/count/bias_mean": -4.994140625, "metrics/particle/count/mae": 6.279296875, "metrics/particle/count/prediction_mean": 65.06185150146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08756495267152786, "metrics/particle/event/energy_response_mean": 1.011620283126831, "metrics/particle/event/met_abs_error": 12.710697174072266, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06911365687847137, "metrics/particle/event/scalar_pt_response_mean": 0.996748685836792, "metrics/particle/event/vector_pt_closure": 0.08826205879449844, "metrics/particle/matched/delta_eta_abs_mean": 0.016885321587324142, "metrics/particle/matched/delta_phi_abs_mean": 0.018167803063988686, "metrics/particle/matched/delta_r_mean": 0.02780337817966938, "metrics/particle/matched/energy_relative_abs_mean": 0.1677699089050293, "metrics/particle/matched/pid_accuracy": 0.9245642423629761, "metrics/particle/matched/pt_relative_abs_mean": 0.16309460997581482, "metrics/particle/matching/duplicate_fraction": 0.08400460332632065, "metrics/particle/matching/efficiency": 0.6882794499397278, "metrics/particle/matching/f1": 0.7137191891670227, "metrics/particle/matching/purity": 0.741111695766449, "metrics/particle/matching/target_coverage_dr0p05": 0.6171402931213379, "metrics/particle/matching/target_coverage_dr0p10": 0.7436667084693909, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.48752856254577637, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7457762360572815, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02689092978835106, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16038262844085693, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1588101089000702, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01123342476785183, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19924694299697876, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010188832879066467, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.866321325302124, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05205095559358597, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.298903703689575, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7361.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3382618725299835, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4491512775421143, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6103988885879517, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.437543272972107, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1822517067193985, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.730363845825195, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02218536101281643, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3380415439605713, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.017581982538104057, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7216637134552, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9646.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6730980277061462, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.4614133834838867, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.088056564331055, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.795782566070557, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 2.2301385402679443, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.0238897800445557, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.211033821105957, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.4615190029144287, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.706016540527344, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02572755329310894, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15830090641975403, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16297295689582825, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02815936505794525, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20379434525966644, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028752155601978302, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.59860098361969, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.041362348943948746, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.949009895324707, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1739.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4553277790546417, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1601333618164062, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.1023693084716797, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.952643632888794, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20181111991405487, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.67548370361328, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06425183266401291, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6901350021362305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06272730976343155, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.4677348136901855, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2319.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5756372213363647, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.9667510986328125, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.319767951965332, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5437159538269043, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.06975003331899643, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.00996454432606697, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.142651557922363, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.442739963531494, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.872413635253906, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028049537912011147, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17079436779022217, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1636790633201599, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.013618381693959236, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20331114530563354, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009185146191157401, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3043891191482544, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02657979354262352, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8768753409385681, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15242.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.33331918716430664, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.9619951248168945, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.4345717430114746, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.2278666496276855, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.18917599320411682, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.149720191955566, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0077663264237344265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6263759732246399, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0030616242438554764, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.5891590118408203, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20473.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.711013674736023, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.681485176086426, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.836152076721191, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.884766578674316, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.795649528503418, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.57673716545105, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.009609699249268, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5046923756599426, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.228201866149902, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02753329463303089, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16675333678722382, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16215626895427704, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.003118601394817233, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20212551951408386, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005710294935852289, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5667757987976074, "metrics/particle/momentum/all/accepted/pz_bias": 0.035338323563337326, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7870973348617554, "metrics/particle/momentum/all/accepted/samples": 24342.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3435114026069641, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7521668672561646, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6060882806777954, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4135221242904663, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1880202293395996, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.083409309387207, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01609223708510399, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9855406284332275, "metrics/particle/momentum/all/assigned/pz_bias": -0.011645016260445118, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.51234769821167, "metrics/particle/momentum/all/assigned/samples": 32438.0, "metrics/particle/momentum/all/efficiency": 0.6877436637878418, "metrics/particle/momentum/all/event/energy_bias": 4.1761474609375, "metrics/particle/momentum/all/event/p3_error_mean": 27.80589485168457, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.674266815185547, "metrics/particle/momentum/all/event/px_bias": 8.095538139343262, "metrics/particle/momentum/all/event/py_bias": -4.590662956237793, "metrics/particle/momentum/all/event/pz_abs_mean": 19.39512825012207, "metrics/particle/momentum/all/event/pz_bias": -4.399566650390625, "metrics/particle/momentum/all/event/pz_residual_rms": 30.019649505615234, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030310338363051414, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16316764056682587, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15751174092292786, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01008332334458828, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19689905643463135, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006049794610589743, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6102460622787476, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03159147500991821, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.748067021369934, "metrics/particle/momentum/class_1/accepted/samples": 12893.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2224877029657364, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6301326155662537, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6895943880081177, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5117523670196533, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.13361960649490356, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.8794779777526855, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011808537878096104, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8667382001876831, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.011380416341125965, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1148736476898193, "metrics/particle/momentum/class_1/assigned/samples": 15836.0, "metrics/particle/momentum/class_1/efficiency": 0.7726373672485352, "metrics/particle/momentum/class_1/event/energy_bias": 15.112995147705078, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.381397247314453, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.531259536743164, "metrics/particle/momentum/class_1/event/px_bias": 5.948058605194092, "metrics/particle/momentum/class_1/event/py_bias": -2.284674644470215, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.663261413574219, "metrics/particle/momentum/class_1/event/pz_bias": -3.079803943634033, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.749881744384766, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033598028123378754, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20511645078659058, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19935506582260132, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.001407208968885243, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.240857794880867, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022823330014944077, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.651006817817688, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13330455124378204, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9432716369628906, "metrics/particle/momentum/class_2/accepted/samples": 1183.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6029840111732483, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8181796073913574, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1381157636642456, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.787234902381897, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3334914743900299, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 11.763992309570312, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05834914743900299, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6463911533355713, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04156545549631119, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.337137699127197, "metrics/particle/momentum/class_2/assigned/samples": 2108.0, "metrics/particle/momentum/class_2/efficiency": 0.39737990498542786, "metrics/particle/momentum/class_2/event/energy_bias": -8.42233657836914, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.276966094970703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.714369773864746, "metrics/particle/momentum/class_2/event/px_bias": 1.8964811563491821, "metrics/particle/momentum/class_2/event/py_bias": -1.2118006944656372, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.442784309387207, "metrics/particle/momentum/class_2/event/pz_bias": -1.4582246541976929, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.52970314025879, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02318958006799221, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16645024716854095, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16357681155204773, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.02134961448609829, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2033487856388092, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002627589739859104, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3470630347728729, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.011634969152510166, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2710745334625244, "metrics/particle/momentum/class_3/accepted/samples": 9895.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4430825114250183, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.1494555473327637, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.704921245574951, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5188405513763428, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22724024951457977, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.618305206298828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01364870648831129, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8386379480361938, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06063356623053551, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.283428430557251, "metrics/particle/momentum/class_3/assigned/samples": 13994.0, "metrics/particle/momentum/class_3/efficiency": 0.6506871581077576, "metrics/particle/momentum/class_3/event/energy_bias": -0.7102121710777283, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.79966926574707, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.4432692527771, "metrics/particle/momentum/class_3/event/px_bias": 0.12314489483833313, "metrics/particle/momentum/class_3/event/py_bias": -1.4033892154693604, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.339500427246094, "metrics/particle/momentum/class_3/event/pz_bias": -0.4819185137748718, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.17448616027832, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02992023155093193, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1706717610359192, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15290139615535736, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04602484405040741, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20016686618328094, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.012605042196810246, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0443499088287354, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.6101481318473816, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.702298879623413, "metrics/particle/momentum/class_4/accepted/samples": 238.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3929482698440552, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.7005720138549805, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3367679119110107, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2140824794769287, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22674418985843658, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.073006629943848, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020348837599158287, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3602935075759888, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.4885197877883911, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.314523696899414, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.6538461446762085, "metrics/particle/momentum/class_4/event/energy_bias": -0.5030006766319275, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.881568193435669, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.634235382080078, "metrics/particle/momentum/class_4/event/px_bias": 0.10575352609157562, "metrics/particle/momentum/class_4/event/py_bias": 0.11795799434185028, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.289510488510132, "metrics/particle/momentum/class_4/event/pz_bias": 0.42724090814590454, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.0089111328125, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.023277156054973602, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18865685164928436, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.192399799823761, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.05474793165922165, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23434039950370789, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03759398311376572, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2005341053009033, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.26205989718437195, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.089455604553223, "metrics/particle/momentum/class_5/accepted/samples": 133.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08170828223228455, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.838525116443634, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.783383846282959, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4995751976966858, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14102564752101898, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.105222702026367, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.954273223876953, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.2235511690378189, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.42962121963501, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8364779949188232, "metrics/particle/momentum/class_5/event/energy_bias": -1.3012981414794922, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7745232582092285, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.058962345123291, "metrics/particle/momentum/class_5/event/px_bias": 0.022100068628787994, "metrics/particle/momentum/class_5/event/py_bias": 0.19124411046504974, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.509682297706604, "metrics/particle/momentum/class_5/event/pz_bias": 0.19313925504684448, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.7090559005737305, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4688637852668762, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4721969962120056, "metrics/set_slot/angular_recall_all_dr0p10": 0.9006561040878296, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7413155436515808, "metrics/set_slot/soft_count_bias_mean": -0.17853884398937225}, "step_train_time": 5.87847900390625, "step_valid_time": 37444.82730937004, "step_total_time": 37450.705788612366, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002300071912233, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20970042746310957, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2822, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2769, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9322761810373307, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9313531353135314, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9318144295856035, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06864686468646865, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9147670961347869, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9138613861386139, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9143140168400198, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9322761810373307, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9948349498696978, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1951928437360012, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2257, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2095, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2073, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9261715296198055, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9282233052724856, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9271962823633547, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0717766947275144, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.916445623342175, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9184758529020824, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9174596149590617, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9261715296198055, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0081165275168464, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19514232639803542, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1261, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1158, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1143, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8990683229813664, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9183187946074544, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9085916045508042, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0816812053925456, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8874223602484472, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9064234734337827, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8968222832483327, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8990683229813664} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_13000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..927974505a89dc39085b565ebe08642afca433f3 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6158802349819974, "train": {"Classification_binary": 0.25137919187545776, "Cardinality": 0.20316265523433685, "Classification": 0.1278306543827057, "Regression_pt": 0.1770976334810257, "Regression_eta": 0.00272889225743711, "Regression_sin_phi": 0.0014990008203312755, "Regression_cos_phi": 0.0014932866906747222, "Regression_energy": 0.1756637692451477, "HitGrouping": 0.18050698935985565, "Auxiliary": 0.2766581177711487, "Total": 1.4749226570129395, "Total_unweighted": 0.73769242875278}, "train_group": {}, "valid": {"Classification_binary": 0.22221614917119345, "Cardinality": 0.15111068884531656, "Classification": 0.10891303420066833, "Regression_pt": 0.17577572663625082, "Regression_eta": 0.002251418928305308, "Regression_sin_phi": 0.001262978029747804, "Regression_cos_phi": 0.0012715549673885107, "Regression_energy": 0.17464635769526163, "HitGrouping": 0.1785194675127665, "Auxiliary": 0.24958926439285278, "Total": 1.3045693238576253, "Total_unweighted": 0.6863372196288159, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8324207067489624, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5018115043640137, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34126758575439453, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5056653618812561, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4934195578098297, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3846226632595062, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.788386344909668, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.48184049129486084, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34598439931869507, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.48400890827178955, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4636005461215973, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4490003287792206, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8456114530563354, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4982205927371979, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3262059688568115, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5001764297485352, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4846705198287964, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3592604100704193, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8362338542938232, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.497906893491745, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33282673358917236, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5004925727844238, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4855959713459015, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3754711151123047, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025758640840649605, "metrics/hit_grouping/calorimeter/background_recall": 0.6346845626831055, "metrics/hit_grouping/calorimeter/completeness": 0.5660072565078735, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5779428482055664, "metrics/hit_grouping/calorimeter/energy_purity": 0.5711259245872498, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39725038409233093, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8595016002655029, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5139244794845581, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26360106468200684, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5197679400444031, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5009298920631409, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3202747106552124, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8339019417762756, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5229796171188354, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3501940667629242, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5237413048744202, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5092064142227173, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3854924738407135, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.789413571357727, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4378749430179596, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4612564742565155, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.43546539545059204, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4285348653793335, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4818606972694397, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8143375515937805, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5159001350402832, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3933976888656616, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.521963894367218, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5208613872528076, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4303955137729645, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8106147050857544, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6115425825119019, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5206698179244995, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6062577962875366, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5886354446411133, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.41339612007141113, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8630520701408386, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4640219509601593, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24367216229438782, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.46387553215026855, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4344981908798218, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3115709125995636, "metrics/hit_grouping/calorimeter/purity": 0.5576533675193787, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8722954988479614, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09597049653530121, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11846154183149338, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09706959873437881, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0638679638504982, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.032307691872119904, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8467671275138855, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.282097727060318, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.13854047656059265, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2795546352863312, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2498813420534134, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2554161846637726, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8348593711853027, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5188862681388855, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.348570317029953, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5217986702919006, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5081421136856079, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.38969045877456665, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7108033299446106, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44082486629486084, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4593009650707245, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48624387383461, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6273161768913269, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7820848226547241, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.36455413699150085, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5043042898178101, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48694005608558655, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4621424674987793, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6449496746063232, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45459556579589844, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38542649149894714, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4142693281173706, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7139481902122498, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6773133873939514, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4426601231098175, "metrics/hit_grouping/tracker/all/object_completeness": 0.4187878966331482, "metrics/hit_grouping/tracker/all/object_purity": 0.44313883781433105, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6653599143028259, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.003190415445715189, "metrics/hit_grouping/tracker/background_recall": 0.025410175323486328, "metrics/hit_grouping/tracker/completeness": 0.43849003314971924, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5189560651779175, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7098768353462219, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42924830317497253, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.49584904313087463, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5075996518135071, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6312528252601624, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6474670767784119, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4557395875453949, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3798924386501312, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4296158254146576, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7114030122756958, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.636738657951355, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4585356116294861, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.28904905915260315, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3144129514694214, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6921520829200745, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.674986720085144, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44769948720932007, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.41520750522613525, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4620858430862427, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6662473082542419, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7019249200820923, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47270306944847107, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4245874583721161, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4484817385673523, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6185086369514465, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6760168075561523, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.43081244826316833, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42243245244026184, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4175163805484772, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6724420785903931, "metrics/hit_grouping/tracker/purity": 0.4172411561012268, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8854386806488037, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.020417029038071632, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.07243099063634872, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.08219961076974869, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08340573310852051, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6629061102867126, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.23691031336784363, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2477361559867859, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2591598331928253, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5909797549247742, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6506730914115906, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.523463785648346, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4853289723396301, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5131334066390991, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7529071569442749, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5054919123649597, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5675119161605835, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6293646097183228, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6050888299942017, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.019764546304941177, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07428404688835144, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.149499893188477, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2971080243587494, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16666823625564575, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8004550933837891, "metrics/particle/class_1/pid_accuracy": 0.9524657130241394, "metrics/particle/class_2/efficiency": 0.4591653645038605, "metrics/particle/class_2/pid_accuracy": 0.660884439945221, "metrics/particle/class_3/efficiency": 0.6761633157730103, "metrics/particle/class_3/pid_accuracy": 0.9527888894081116, "metrics/particle/class_4/efficiency": 0.6669874787330627, "metrics/particle/class_4/pid_accuracy": 0.3968254029750824, "metrics/particle/class_5/efficiency": 0.8899999856948853, "metrics/particle/class_5/pid_accuracy": 0.5325842499732971, "metrics/particle/count/bias_mean": -3.4205729961395264, "metrics/particle/count/mae": 5.173177242279053, "metrics/particle/count/prediction_mean": 66.63541412353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08567553758621216, "metrics/particle/event/energy_response_mean": 0.9571893811225891, "metrics/particle/event/met_abs_error": 8.890663146972656, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07065301388502121, "metrics/particle/event/scalar_pt_response_mean": 0.9762184619903564, "metrics/particle/event/vector_pt_closure": 0.06720737367868423, "metrics/particle/matched/delta_eta_abs_mean": 0.016326753422617912, "metrics/particle/matched/delta_phi_abs_mean": 0.017580175772309303, "metrics/particle/matched/delta_r_mean": 0.02687935344874859, "metrics/particle/matched/energy_relative_abs_mean": 0.1561645120382309, "metrics/particle/matched/pid_accuracy": 0.9297316074371338, "metrics/particle/matched/pt_relative_abs_mean": 0.15478889644145966, "metrics/particle/matching/duplicate_fraction": 0.07758519798517227, "metrics/particle/matching/efficiency": 0.7174692749977112, "metrics/particle/matching/f1": 0.7354232668876648, "metrics/particle/matching/purity": 0.7542988657951355, "metrics/particle/matching/target_coverage_dr0p05": 0.6385145783424377, "metrics/particle/matching/target_coverage_dr0p10": 0.7664628624916077, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5277308225631714, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.776741087436676, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025924907997250557, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1530809998512268, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15212823450565338, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02416970767080784, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19061143696308136, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009226038120687008, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8296880722045898, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.13210953772068024, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1533043384552, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7804.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.35323116183280945, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2702362537384033, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4140574932098389, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2082618474960327, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16351136565208435, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.672619819641113, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018632922321558, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2381352186203003, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05253497511148453, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3727383613586426, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10036.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7136064171791077, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.3662240505218506, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.043282508850098, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.673941612243652, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.10375360399484634, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.26347702741622925, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.442811965942383, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5049355030059814, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.151819229125977, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024437492713332176, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16115406155586243, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15766651928424835, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.026036590337753296, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19546695053577423, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03523758798837662, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7265770435333252, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11087454110383987, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.636901378631592, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1873.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5086132287979126, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2880409955978394, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.076411485671997, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.8652780055999756, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.191996768116951, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 27.963607788085938, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06709782034158707, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7384185791015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.047578148543834686, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.9811692237854, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2474.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6199933886528015, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.204895973205566, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.158679962158203, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5184788703918457, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06328979134559631, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04606235772371292, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.975793838500977, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.8743863105773926, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.74341583251953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027466438710689545, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15592731535434723, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15484404563903809, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.015932215377688408, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19414134323596954, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009517162689007819, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2859607934951782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03498247265815735, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8559812903404236, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15761.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3115398585796356, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6535903215408325, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2828316688537598, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0719705820083618, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.17212876677513123, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.724653244018555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006767208222299814, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5717605352401733, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.011439683847129345, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3267455101013184, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20688.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7352241277694702, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.3788239657878876, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.57735824584961, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.721799850463867, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.39533311128616333, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.252336025238037, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.249950885772705, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.03413308411836624, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.344378471374512, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026770496740937233, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15543894469738007, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15421870350837708, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.019203340634703636, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19316373765468597, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006014623679220676, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.558840811252594, "metrics/particle/momentum/all/accepted/pz_bias": 0.07036754488945007, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8599883317947388, "metrics/particle/momentum/all/accepted/samples": 25438.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3388298451900482, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5104578733444214, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4561643600463867, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.246814250946045, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1710042804479599, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.821235656738281, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014850292354822159, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.934675395488739, "metrics/particle/momentum/all/assigned/pz_bias": 0.0052071972750127316, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.399934768676758, "metrics/particle/momentum/all/assigned/samples": 33198.0, "metrics/particle/momentum/all/efficiency": 0.718709409236908, "metrics/particle/momentum/all/event/energy_bias": -11.949944496154785, "metrics/particle/momentum/all/event/p3_error_mean": 23.160036087036133, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.875456809997559, "metrics/particle/momentum/all/event/px_bias": 0.435796856880188, "metrics/particle/momentum/all/event/py_bias": 2.4697508811950684, "metrics/particle/momentum/all/event/pz_abs_mean": 16.86509895324707, "metrics/particle/momentum/all/event/pz_bias": -2.413455009460449, "metrics/particle/momentum/all/event/pz_residual_rms": 25.9818172454834, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030534932389855385, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14645011723041534, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1451433151960373, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.01849759742617607, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1833379715681076, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006490599829703569, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5949168801307678, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.09696561098098755, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8397330045700073, "metrics/particle/momentum/class_1/accepted/samples": 13404.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2139378935098648, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5028051733970642, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5802145600318909, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3805125951766968, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11264711618423462, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.450666904449463, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011644560843706131, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.819765031337738, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06243234500288963, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.9225242137908936, "metrics/particle/momentum/class_1/assigned/samples": 16059.0, "metrics/particle/momentum/class_1/efficiency": 0.8032600283622742, "metrics/particle/momentum/class_1/event/energy_bias": 2.213639259338379, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.703922271728516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.672345161437988, "metrics/particle/momentum/class_1/event/px_bias": 0.18792393803596497, "metrics/particle/momentum/class_1/event/py_bias": 1.2558752298355103, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.607331275939941, "metrics/particle/momentum/class_1/event/pz_bias": 0.08822011947631836, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.711509704589844, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03243045136332512, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1952463835477829, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19687744975090027, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.025609439238905907, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23876042664051056, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0245535708963871, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.668959140777588, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.052472762763500214, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9632155895233154, "metrics/particle/momentum/class_2/accepted/samples": 1344.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6622119545936584, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6953721046447754, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9356244206428528, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5444083213806152, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2996941804885864, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.3498334884643555, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05373525619506836, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5958163738250732, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.026716312393546104, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.564143180847168, "metrics/particle/momentum/class_2/assigned/samples": 2289.0, "metrics/particle/momentum/class_2/efficiency": 0.45146119594573975, "metrics/particle/momentum/class_2/event/energy_bias": -10.851435661315918, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.176729202270508, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.237752914428711, "metrics/particle/momentum/class_2/event/px_bias": 0.6535202264785767, "metrics/particle/momentum/class_2/event/py_bias": 0.46010109782218933, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.415083885192871, "metrics/particle/momentum/class_2/event/pz_bias": -2.516395092010498, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.5518856048584, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02116457186639309, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16141784191131592, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1599116176366806, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.019928758963942528, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19816666841506958, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002330097137019038, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.33077120780944824, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.022601809352636337, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2914572954177856, "metrics/particle/momentum/class_3/accepted/samples": 10300.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.42739567160606384, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.7575807571411133, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.5071308612823486, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.315415382385254, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2152269333600998, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.2977237701416, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01185247115790844, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7726625204086304, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.07702985405921936, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.1261532306671143, "metrics/particle/momentum/class_3/assigned/samples": 14343.0, "metrics/particle/momentum/class_3/efficiency": 0.6773196458816528, "metrics/particle/momentum/class_3/event/energy_bias": -0.8793389201164246, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.011545181274414, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.035004138946533, "metrics/particle/momentum/class_3/event/px_bias": -0.13020652532577515, "metrics/particle/momentum/class_3/event/py_bias": 0.4875480830669403, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.904056549072266, "metrics/particle/momentum/class_3/event/pz_bias": -0.5821563005447388, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.219009399414062, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027591997757554054, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15494410693645477, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1535293161869049, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.047783322632312775, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19638992846012115, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.012096773833036423, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1182091236114502, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.5839937329292297, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6643242835998535, "metrics/particle/momentum/class_4/accepted/samples": 248.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.436175674200058, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.513176679611206, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.508084535598755, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.373474359512329, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24079319834709167, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.403512001037598, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01133144460618496, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.333337426185608, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.6419662833213806, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.5356483459472656, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.6813187003135681, "metrics/particle/momentum/class_4/event/energy_bias": -1.057077407836914, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.063502073287964, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0597646236419678, "metrics/particle/momentum/class_4/event/px_bias": 0.011822454631328583, "metrics/particle/momentum/class_4/event/py_bias": -0.13781432807445526, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9339560270309448, "metrics/particle/momentum/class_4/event/pz_bias": 0.6464214324951172, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.258366584777832, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.023051617667078972, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1943489909172058, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1953972578048706, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08956146240234375, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2364530712366104, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0422535203397274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2125773429870605, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.29668989777565, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.268325328826904, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08399256318807602, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.25192150473594666, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24222242832183838, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.08376865088939667, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06493506580591202, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.36714082956314087, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05844155699014664, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.4023163318634033, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08221371471881866, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.684159755706787, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.893081784248352, "metrics/particle/momentum/class_5/event/energy_bias": -1.375731110572815, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.633357286453247, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.055704355239868, "metrics/particle/momentum/class_5/event/px_bias": -0.2872631847858429, "metrics/particle/momentum/class_5/event/py_bias": 0.4040403962135315, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3197129964828491, "metrics/particle/momentum/class_5/event/pz_bias": -0.0495450496673584, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.017312526702881, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.443847119808197, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.44574594497680664, "metrics/set_slot/angular_recall_all_dr0p10": 0.912830114364624, "metrics/set_slot/angular_recall_selected_dr0p10": 0.76064532995224, "metrics/set_slot/soft_count_bias_mean": 1.3841062784194946}, "step_train_time": 5.739274501800537, "step_valid_time": 40501.23370933533, "step_total_time": 40506.972984313965, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9908251964861756, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18039837970600192, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3014, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2837, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2801, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372315824248431, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412740544127406, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9392484687965569, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05872594558725946, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9253386190948133, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.929329794293298, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9273299122661811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372315824248431, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9654071202339515, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16495918537355803, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2098, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2085, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9274977895667551, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391226499552372, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9332740213523132, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06087735004476276, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9217506631299734, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9333034914950761, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.927491103202847, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9274977895667551, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9715653090454252, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19426235743581988, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1256, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1161, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1153, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9013975155279503, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9243630573248408, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9127358490566038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07563694267515923, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8951863354037267, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9179936305732485, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9064465408805031, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9013975155279503} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_14000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..7b99d3ed7c01ad0e5a760a050c5636fce7370b50 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6632556376729202, "train": {"Classification_binary": 0.2520769238471985, "Cardinality": 0.22330781817436218, "Classification": 0.13499943912029266, "Regression_pt": 0.18265880644321442, "Regression_eta": 0.0023487377911806107, "Regression_sin_phi": 0.0014294555876404047, "Regression_cos_phi": 0.0014192807720974088, "Regression_energy": 0.18152418732643127, "HitGrouping": 0.18198806047439575, "Auxiliary": 0.2922303080558777, "Total": 1.5016034841537476, "Total_unweighted": 0.7564568308880553}, "train_group": {}, "valid": {"Classification_binary": 0.22014365593592325, "Cardinality": 0.14861876765886942, "Classification": 0.10633031527201335, "Regression_pt": 0.16710313161214194, "Regression_eta": 0.0023175207898020744, "Regression_sin_phi": 0.0012463520591457684, "Regression_cos_phi": 0.0012652385048568249, "Regression_energy": 0.16599491238594055, "HitGrouping": 0.17816948890686035, "Auxiliary": 0.24610581000645956, "Total": 1.2856325308481853, "Total_unweighted": 0.6644011265598238, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8312370777130127, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5042782425880432, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33512797951698303, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5080582499504089, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4962102472782135, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3892509639263153, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7780126333236694, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.48619455099105835, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34056252241134644, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4919794201850891, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4803529977798462, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4490003287792206, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8505136370658875, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4960610568523407, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3194401264190674, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4974778890609741, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.47820019721984863, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3524429202079773, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8378074169158936, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.49782034754753113, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32638296484947205, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5003893971443176, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4841887950897217, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3729483187198639, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026234298944473267, "metrics/hit_grouping/calorimeter/background_recall": 0.6389411687850952, "metrics/hit_grouping/calorimeter/completeness": 0.5658331513404846, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5792075395584106, "metrics/hit_grouping/calorimeter/energy_purity": 0.5726960897445679, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3916754126548767, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8569884300231934, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5201861262321472, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26431772112846375, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5268723964691162, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5064820051193237, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.31884145736694336, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8393885493278503, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5175135731697083, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3360019326210022, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.517948567867279, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5057960748672485, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.383551687002182, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7955598831176758, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4299571216106415, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44773101806640625, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4260351061820984, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4144846796989441, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4764252305030823, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8210691213607788, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5184890031814575, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.38411709666252136, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5232692360877991, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5164155960083008, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.42030519247055054, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8207651376724243, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6086463332176208, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5232862234115601, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6058731079101562, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5720437169075012, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.39141809940338135, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8579829335212708, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.46141859889030457, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2391187846660614, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.46229398250579834, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.438300758600235, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.31967324018478394, "metrics/hit_grouping/calorimeter/purity": 0.5577540993690491, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8526337146759033, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11343733221292496, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13230769336223602, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11283699423074722, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06762487441301346, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04153846204280853, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8465784192085266, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.27303972840309143, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12998859584331512, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2728768289089203, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.24290774762630463, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2440136820077896, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8369869589805603, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5189415812492371, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.341815322637558, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5217353105545044, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.506945013999939, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3874278664588928, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.711432158946991, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43126121163368225, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4703126847743988, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4985846281051636, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6196951866149902, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7909266352653503, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3595064580440521, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5295639634132385, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5139747858047485, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44643858075141907, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.645375669002533, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4479798376560211, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3848661780357361, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.41513046622276306, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.709222674369812, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6785122752189636, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4352446496486664, "metrics/hit_grouping/tracker/all/object_completeness": 0.4240596294403076, "metrics/hit_grouping/tracker/all/object_purity": 0.4498326778411865, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.658794641494751, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0037455353885889053, "metrics/hit_grouping/tracker/background_recall": 0.02458093874156475, "metrics/hit_grouping/tracker/completeness": 0.43978917598724365, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.48687848448753357, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7074627876281738, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4275170862674713, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5056051015853882, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5229175686836243, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6256036162376404, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6497222781181335, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43993908166885376, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38383713364601135, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4320186674594879, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7034075856208801, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6448352336883545, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4474110007286072, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2857763171195984, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.30652156472206116, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6850728392601013, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6821813583374023, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44473400712013245, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4198667109012604, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4560079872608185, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.651509702205658, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6883602738380432, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4420772194862366, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.39495494961738586, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.45241597294807434, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6398136019706726, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6720007061958313, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.42168110609054565, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4346396028995514, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.441333532333374, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6716225743293762, "metrics/hit_grouping/tracker/purity": 0.41871052980422974, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8832136988639832, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.026498697698116302, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.09397201240062714, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.10545443743467331, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0803648978471756, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6612665057182312, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2555728256702423, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2761858105659485, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2831845283508301, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5935717821121216, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6526604890823364, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5112448930740356, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48575082421302795, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5156034231185913, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.744820237159729, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5246052742004395, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5745446085929871, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6382986307144165, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.613956093788147, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0388779491186142, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08315134793519974, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.814247131347656, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28767502307891846, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15723520517349243, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8024337291717529, "metrics/particle/class_1/pid_accuracy": 0.9617063403129578, "metrics/particle/class_2/efficiency": 0.4375140368938446, "metrics/particle/class_2/pid_accuracy": 0.6223077178001404, "metrics/particle/class_3/efficiency": 0.6432111263275146, "metrics/particle/class_3/pid_accuracy": 0.9478687047958374, "metrics/particle/class_4/efficiency": 0.6525505185127258, "metrics/particle/class_4/pid_accuracy": 0.3687315583229065, "metrics/particle/class_5/efficiency": 0.8679999709129333, "metrics/particle/class_5/pid_accuracy": 0.47235023975372314, "metrics/particle/count/bias_mean": -4.723307132720947, "metrics/particle/count/mae": 5.844400882720947, "metrics/particle/count/prediction_mean": 65.33267974853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08991450816392899, "metrics/particle/event/energy_response_mean": 1.0023375749588013, "metrics/particle/event/met_abs_error": 9.429862022399902, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07427369058132172, "metrics/particle/event/scalar_pt_response_mean": 1.0175812244415283, "metrics/particle/event/vector_pt_closure": 0.07398632168769836, "metrics/particle/matched/delta_eta_abs_mean": 0.01880771480500698, "metrics/particle/matched/delta_phi_abs_mean": 0.018100883811712265, "metrics/particle/matched/delta_r_mean": 0.02921261265873909, "metrics/particle/matched/energy_relative_abs_mean": 0.15783119201660156, "metrics/particle/matched/pid_accuracy": 0.9305616021156311, "metrics/particle/matched/pt_relative_abs_mean": 0.15678444504737854, "metrics/particle/matching/duplicate_fraction": 0.08356668055057526, "metrics/particle/matching/efficiency": 0.7020891308784485, "metrics/particle/matching/f1": 0.7265828847885132, "metrics/particle/matching/purity": 0.7528474926948547, "metrics/particle/matching/target_coverage_dr0p05": 0.6204208135604858, "metrics/particle/matching/target_coverage_dr0p10": 0.7621694207191467, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5125736594200134, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7571139335632324, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027440952137112617, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1506016105413437, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15346086025238037, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03581741079688072, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19424259662628174, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009894459508359432, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8179535865783691, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.045992035418748856, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1929500102996826, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7580.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32043325901031494, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3920117616653442, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7270383834838867, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5829881429672241, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16618527472019196, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 20.838069915771484, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019496595486998558, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2195969820022583, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.007901686243712902, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3737800121307373, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9694.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6931236386299133, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5386953353881836, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.890989303588867, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.796207904815674, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.9896814227104187, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.12095268815755844, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.398942947387695, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.260756492614746, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.072887420654297, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02584567293524742, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1484079211950302, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15347345173358917, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03823419287800789, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19430333375930786, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028713418170809746, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5461888313293457, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.21693985164165497, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.120965957641602, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1811.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4092576801776886, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.096933364868164, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.5904595851898193, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.446589946746826, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19394195079803467, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.57311248779297, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06604964286088943, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5845115184783936, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06664909422397614, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.029238224029541, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2377.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5994703769683838, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.09799861907959, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.72171688079834, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5954912900924683, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.3230553865432739, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.016011234372854233, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.532630920410156, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.3511457443237305, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.63077735900879, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.030071571469306946, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16122335195541382, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15739794075489044, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.03488686680793762, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19871991872787476, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010371427051723003, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.29132142663002014, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02114492654800415, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8374723196029663, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15427.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.30039000511169434, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7885291576385498, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.4810748100280762, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.3317341804504395, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.18490786850452423, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.423004150390625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006466669961810112, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5759220123291016, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0292743481695652, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.20526385307312, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20567.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7196435928344727, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 8.779803276062012, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.094731330871582, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.294844627380371, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -3.886411190032959, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4006842076778412, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.281294822692871, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.05787336826324463, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.864530563354492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02895975299179554, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15704406797885895, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15590907633304596, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03541533648967743, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1970420628786087, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005761947017163038, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.543736457824707, "metrics/particle/momentum/all/accepted/pz_bias": 0.016733618453145027, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7731281518936157, "metrics/particle/momentum/all/accepted/samples": 24818.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3142718970775604, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6203889846801758, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6349256038665771, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4875547885894775, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.18000490963459015, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.829761505126953, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01467614434659481, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9133875966072083, "metrics/particle/momentum/all/assigned/pz_bias": -0.01594029925763607, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1691982746124268, "metrics/particle/momentum/all/assigned/samples": 32638.0, "metrics/particle/momentum/all/efficiency": 0.701192319393158, "metrics/particle/momentum/all/event/energy_bias": 1.1431100368499756, "metrics/particle/momentum/all/event/p3_error_mean": 24.97100830078125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.771842956542969, "metrics/particle/momentum/all/event/px_bias": -5.199148178100586, "metrics/particle/momentum/all/event/py_bias": 0.26372021436691284, "metrics/particle/momentum/all/event/pz_abs_mean": 18.21055030822754, "metrics/particle/momentum/all/event/pz_bias": -4.5540289878845215, "metrics/particle/momentum/all/event/pz_residual_rms": 28.113956451416016, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.033585622906684875, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1481594443321228, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14946627616882324, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04009979963302612, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18971748650074005, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004934948403388262, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.560800313949585, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0025838136207312346, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6378743648529053, "metrics/particle/momentum/class_1/accepted/samples": 13374.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18155351281166077, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5569053292274475, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.623930037021637, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.49516159296035767, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11238905042409897, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.569179534912109, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01000125054270029, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7633745670318604, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005834433250129223, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.52365779876709, "metrics/particle/momentum/class_1/assigned/samples": 15998.0, "metrics/particle/momentum/class_1/efficiency": 0.8014622330665588, "metrics/particle/momentum/class_1/event/energy_bias": 23.869569778442383, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.72614288330078, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.281988143920898, "metrics/particle/momentum/class_1/event/px_bias": -4.109620571136475, "metrics/particle/momentum/class_1/event/py_bias": 1.1346336603164673, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.329193115234375, "metrics/particle/momentum/class_1/event/pz_bias": -2.009295701980591, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.467918395996094, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03306630253791809, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19776569306850433, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1983896940946579, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.01846015825867653, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23768484592437744, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.640599250793457, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0763271376490593, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8976128101348877, "metrics/particle/momentum/class_2/accepted/samples": 1292.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6296466588973999, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7958706617355347, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.155774712562561, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8139461874961853, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2834498882293701, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.622167110443115, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05967365950345993, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.686229705810547, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.16717295348644257, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.0871100425720215, "metrics/particle/momentum/class_2/assigned/samples": 2145.0, "metrics/particle/momentum/class_2/efficiency": 0.43399396538734436, "metrics/particle/momentum/class_2/event/energy_bias": -19.607343673706055, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.8007755279541, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.138328552246094, "metrics/particle/momentum/class_2/event/px_bias": 1.1078017950057983, "metrics/particle/momentum/class_2/event/py_bias": -1.0500819683074951, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.224400520324707, "metrics/particle/momentum/class_2/event/pz_bias": 0.23277059197425842, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.283737182617188, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022086570039391518, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16325591504573822, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1586592197418213, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03700301796197891, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20044970512390137, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003174278186634183, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34096837043762207, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.025621945038437843, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3839317560195923, "metrics/particle/momentum/class_3/accepted/samples": 9766.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4189303517341614, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.9652810096740723, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.8628346920013428, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.7246639728546143, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.240982785820961, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.267494201660156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012284836731851101, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7890358567237854, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06468179076910019, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.822329044342041, "metrics/particle/momentum/class_3/assigned/samples": 14001.0, "metrics/particle/momentum/class_3/efficiency": 0.642204225063324, "metrics/particle/momentum/class_3/event/energy_bias": -0.8316347002983093, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.225162506103516, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.812197208404541, "metrics/particle/momentum/class_3/event/px_bias": -1.621347188949585, "metrics/particle/momentum/class_3/event/py_bias": -0.21325744688510895, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.598634719848633, "metrics/particle/momentum/class_3/event/pz_bias": -2.597223997116089, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.35480499267578, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0297224223613739, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16202412545681, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15692704916000366, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03829214721918106, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2010490596294403, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02024291455745697, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.029542088508606, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4984017312526703, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.4875714778900146, "metrics/particle/momentum/class_4/accepted/samples": 247.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.37085437774658203, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1025843620300293, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.295468807220459, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1564266681671143, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23529411852359772, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.401673316955566, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01764705963432789, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.109741449356079, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.4202725887298584, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.579946279525757, "metrics/particle/momentum/class_4/assigned/samples": 340.0, "metrics/particle/momentum/class_4/efficiency": 0.6785714030265808, "metrics/particle/momentum/class_4/event/energy_bias": -1.0640628337860107, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.277709484100342, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.065842866897583, "metrics/particle/momentum/class_4/event/px_bias": 0.022442152723670006, "metrics/particle/momentum/class_4/event/py_bias": -0.02203715592622757, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1299519538879395, "metrics/particle/momentum/class_4/event/pz_bias": 0.1116696447134018, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.873848915100098, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.027254801243543625, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1880900263786316, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1859237551689148, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.031194021925330162, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22172234952449799, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.050359711050987244, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.089643955230713, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6561495661735535, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.058604717254639, "metrics/particle/momentum/class_5/accepted/samples": 139.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06874421238899231, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2465379685163498, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.23976366221904755, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013622822239995003, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09740259498357773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.32602742314338684, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08441558480262756, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.676056385040283, "metrics/particle/momentum/class_5/assigned/pz_bias": -1.3601797819137573, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.3424601554870605, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.8742138147354126, "metrics/particle/momentum/class_5/event/energy_bias": -1.2234183549880981, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.801286458969116, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.095163345336914, "metrics/particle/momentum/class_5/event/px_bias": -0.5984244346618652, "metrics/particle/momentum/class_5/event/py_bias": 0.41446301341056824, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.5283029079437256, "metrics/particle/momentum/class_5/event/pz_bias": -0.2919495105743408, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.991641521453857, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.44893380999565125, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4515364468097687, "metrics/set_slot/angular_recall_all_dr0p10": 0.9147352576255798, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7580711245536804, "metrics/set_slot/soft_count_bias_mean": -0.9163612723350525}, "step_train_time": 6.057629585266113, "step_valid_time": 43586.269354104996, "step_total_time": 43592.32698392868, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0318365330537, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19465476672403703, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2835, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2797, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9365708622398414, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9371900826446281, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9368803701255783, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0628099173553719, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.92401717872481, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9246280991735537, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9243225380039656, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9365708622398414, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0154076910317194, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18293009298977025, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2102, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9350132625994695, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408362989323843, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9379157427937915, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059163701067615655, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9292661361626879, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9350533807829181, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9321507760532151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9350132625994695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.027168570927759, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20255960789567384, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1170, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1158, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9083850931677019, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9169278996865203, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9126365054602185, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08307210031347963, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8990683229813664, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9075235109717869, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9032761310452418, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9083850931677019} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_15000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..540c4baf19a44ed04eb19b1ccf3ab8c0bd078c33 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7106310403638431, "train": {"Classification_binary": 0.23636122047901154, "Cardinality": 0.17067834734916687, "Classification": 0.10034042596817017, "Regression_pt": 0.1707204431295395, "Regression_eta": 0.0024718022905290127, "Regression_sin_phi": 0.0012388061732053757, "Regression_cos_phi": 0.001330338534899056, "Regression_energy": 0.16996356844902039, "HitGrouping": 0.17874541878700256, "Auxiliary": 0.2574177384376526, "Total": 1.3450342416763306, "Total_unweighted": 0.682426605024375}, "train_group": {}, "valid": {"Classification_binary": 0.21772857507069907, "Cardinality": 0.1627017855644226, "Classification": 0.10118078192075093, "Regression_pt": 0.16605940461158752, "Regression_eta": 0.002220028080046177, "Regression_sin_phi": 0.0012130255345255136, "Regression_cos_phi": 0.0012235256532828014, "Regression_energy": 0.16444537043571472, "HitGrouping": 0.17745667695999146, "Auxiliary": 0.25021539131800336, "Total": 1.2879135608673096, "Total_unweighted": 0.6540707113066068, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8293919563293457, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5080086588859558, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.345140278339386, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5120969414710999, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.499274879693985, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.39142343401908875, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7819684147834778, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4883197546005249, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.354794979095459, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4931565821170807, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4920932352542877, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.45035579800605774, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8425220847129822, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4972046911716461, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.31933683156967163, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4996657073497772, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.48699575662612915, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.35895052552223206, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8328654766082764, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4998844265937805, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3308206796646118, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5030821561813354, "metrics/hit_grouping/calorimeter/all/object_purity": 0.49140432476997375, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3775987923145294, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026369495317339897, "metrics/hit_grouping/calorimeter/background_recall": 0.6415259838104248, "metrics/hit_grouping/calorimeter/completeness": 0.5733502507209778, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5802042484283447, "metrics/hit_grouping/calorimeter/energy_purity": 0.5735432505607605, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39832940697669983, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8507825136184692, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5110763311386108, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2618692219257355, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.517782986164093, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.503083348274231, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32547029852867126, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8373149037361145, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5282084345817566, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3503153920173645, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5297694802284241, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5136755108833313, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.38306647539138794, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7903041243553162, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.44667840003967285, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45645302534103394, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.44415444135665894, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4434749484062195, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4822399318218231, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8171612620353699, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5268142223358154, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3961382806301117, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5338737964630127, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5325270295143127, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4256618022918701, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8127363920211792, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.613422691822052, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5253793597221375, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6094419360160828, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5945010185241699, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.41705912351608276, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8523241877555847, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.456404447555542, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23570376634597778, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4563691020011902, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.43400219082832336, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3208785355091095, "metrics/hit_grouping/calorimeter/purity": 0.565242350101471, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.84574294090271, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1120230183005333, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10461538285017014, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11136264353990555, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07484234869480133, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.035384614020586014, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8154991865158081, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2525025010108948, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11687571555376053, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.24927565455436707, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.23081748187541962, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24002280831336975, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8335898518562317, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5223797559738159, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.34794726967811584, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5260292291641235, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5152708888053894, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3928056061267853, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7111499905586243, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4342498481273651, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4662853181362152, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.502210259437561, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6282128095626831, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7804006934165955, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3454851508140564, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5287409424781799, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5400668978691101, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46719011664390564, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6522166728973389, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4457745850086212, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3906969130039215, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4209577441215515, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7011892795562744, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6816392540931702, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4336859881877899, "metrics/hit_grouping/tracker/all/object_completeness": 0.4262140989303589, "metrics/hit_grouping/tracker/all/object_purity": 0.4566706418991089, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.658416748046875, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004413509741425514, "metrics/hit_grouping/tracker/background_recall": 0.026239411905407906, "metrics/hit_grouping/tracker/completeness": 0.4438345432281494, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4997580349445343, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7139014005661011, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4156719744205475, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5016735792160034, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5223407745361328, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6227790713310242, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6522741913795471, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4549781084060669, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3890237808227539, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.43973308801651, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7016942501068115, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6412221193313599, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4510517716407776, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2982192039489746, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.32888805866241455, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.691545307636261, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6803857684135437, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44778937101364136, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42509278655052185, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.47128430008888245, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6566319465637207, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7020565867424011, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.457390159368515, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41794535517692566, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4510023295879364, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6225033402442932, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6796821355819702, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4111449420452118, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4291289448738098, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.43862950801849365, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6670569181442261, "metrics/hit_grouping/tracker/purity": 0.4228455126285553, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8886653780937195, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.025629887357354164, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.08419886231422424, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0984271690249443, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08384013921022415, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6692587733268738, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24883359670639038, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2740904986858368, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2811754643917084, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5821669101715088, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6549175977706909, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5101823806762695, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49001002311706543, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5253328680992126, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7451744079589844, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.516171932220459, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5764079689979553, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6398875713348389, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6213189959526062, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.030444668605923653, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09051428735256195, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.055810928344727, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2743484079837799, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14390860497951508, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7976256608963013, "metrics/particle/class_1/pid_accuracy": 0.9583250880241394, "metrics/particle/class_2/efficiency": 0.4384114742279053, "metrics/particle/class_2/pid_accuracy": 0.6407369375228882, "metrics/particle/class_3/efficiency": 0.6517065763473511, "metrics/particle/class_3/pid_accuracy": 0.9531568884849548, "metrics/particle/class_4/efficiency": 0.6361886262893677, "metrics/particle/class_4/pid_accuracy": 0.3751890957355499, "metrics/particle/class_5/efficiency": 0.8259999752044678, "metrics/particle/class_5/pid_accuracy": 0.5302663445472717, "metrics/particle/count/bias_mean": -5.82421875, "metrics/particle/count/mae": 6.509114742279053, "metrics/particle/count/prediction_mean": 64.23177337646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08112633973360062, "metrics/particle/event/energy_response_mean": 0.9861980080604553, "metrics/particle/event/met_abs_error": 8.798943519592285, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06509047746658325, "metrics/particle/event/scalar_pt_response_mean": 1.0058298110961914, "metrics/particle/event/vector_pt_closure": 0.06821147352457047, "metrics/particle/matched/delta_eta_abs_mean": 0.01758602075278759, "metrics/particle/matched/delta_phi_abs_mean": 0.017532486468553543, "metrics/particle/matched/delta_r_mean": 0.027856208384037018, "metrics/particle/matched/energy_relative_abs_mean": 0.15634635090827942, "metrics/particle/matched/pid_accuracy": 0.9324188828468323, "metrics/particle/matched/pt_relative_abs_mean": 0.15692941844463348, "metrics/particle/matching/duplicate_fraction": 0.06786944717168808, "metrics/particle/matching/efficiency": 0.7032321691513062, "metrics/particle/matching/f1": 0.7337321639060974, "metrics/particle/matching/purity": 0.766997754573822, "metrics/particle/matching/target_coverage_dr0p05": 0.6248629093170166, "metrics/particle/matching/target_coverage_dr0p10": 0.756426215171814, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5107522010803223, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.756928026676178, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02646523527801037, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14969101548194885, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15361455082893372, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03888989984989166, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19430845975875854, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009402727708220482, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8219863176345825, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.12515904009342194, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.135650634765625, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7551.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29211699962615967, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.109572410583496, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2257838249206543, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.082848310470581, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15877321362495422, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.418492317199707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01919465884566307, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2194143533706665, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09701680392026901, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.52960467338562, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9586.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6904718279838562, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3233743906021118, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.900481224060059, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.009851455688477, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6468657851219177, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.30673834681510925, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.33197021484375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.7944843769073486, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.273502349853516, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02460857667028904, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15285438299179077, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1583244502544403, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04756571725010872, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19767358899116516, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029281767085194588, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5973604917526245, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.24824954569339752, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.14695405960083, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1810.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.385846883058548, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0731096267700195, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.9785430431365967, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.844374179840088, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17006507515907288, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 43.306175231933594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05379609391093254, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.470951795578003, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.33942222595214844, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.837384223937988, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2305.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5991393327713013, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.010593891143799, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.949897766113281, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4296784400939941, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11747460067272186, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06635114550590515, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.801515579223633, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.370108038187027, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.526884078979492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028587706387043, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15985289216041565, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1584734469652176, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019874176010489464, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19944828748703003, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011591962538659573, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.28663456439971924, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.017124315723776817, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.845395565032959, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15528.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3103858530521393, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.589563250541687, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1798436641693115, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0035927295684814, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1739194542169571, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.635499954223633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006834832951426506, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5748345851898193, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -3.0038889235584065e-05, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.4595835208892822, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20337.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7243551015853882, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.0707669258117676, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.009148597717285, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.286069869995117, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.517435312271118, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9329502582550049, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.990281581878662, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.09365355968475342, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.58891773223877, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02765440195798874, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15626096725463867, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1569884717464447, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.027657106518745422, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1977735012769699, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005705331452190876, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.544373095035553, "metrics/particle/momentum/all/accepted/pz_bias": -0.06670874357223511, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7550703287124634, "metrics/particle/momentum/all/accepted/samples": 24889.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3103490173816681, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4098554849624634, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.32215416431427, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1588225364685059, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16913864016532898, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.21518325805664, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01386992633342743, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9021739363670349, "metrics/particle/momentum/all/assigned/pz_bias": -0.053151991218328476, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4519922733306885, "metrics/particle/momentum/all/assigned/samples": 32228.0, "metrics/particle/momentum/all/efficiency": 0.703198254108429, "metrics/particle/momentum/all/event/energy_bias": -3.2632012367248535, "metrics/particle/momentum/all/event/p3_error_mean": 23.927223205566406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.77029037475586, "metrics/particle/momentum/all/event/px_bias": 3.0468268394470215, "metrics/particle/momentum/all/event/py_bias": -0.5598607063293457, "metrics/particle/momentum/all/event/pz_abs_mean": 18.00965118408203, "metrics/particle/momentum/all/event/pz_bias": -3.2582457065582275, "metrics/particle/momentum/all/event/pz_residual_rms": 29.94039535522461, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03083399310708046, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1490333080291748, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15004248917102814, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.042496927082538605, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.190031498670578, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005310793407261372, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5678247809410095, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.08824468404054642, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6169559955596924, "metrics/particle/momentum/class_1/accepted/samples": 13369.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1826850026845932, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.503373920917511, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4764673709869385, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.33594897389411926, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10948031395673752, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.891168117523193, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009889763779938221, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7754848599433899, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.05616138502955437, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.911398410797119, "metrics/particle/momentum/class_1/assigned/samples": 15875.0, "metrics/particle/momentum/class_1/efficiency": 0.801162600517273, "metrics/particle/momentum/class_1/event/energy_bias": 14.718460083007812, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.374168395996094, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.074365615844727, "metrics/particle/momentum/class_1/event/px_bias": 2.5521740913391113, "metrics/particle/momentum/class_1/event/py_bias": 0.18773862719535828, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.098762512207031, "metrics/particle/momentum/class_1/event/pz_bias": 0.7185802459716797, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.790048599243164, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032251253724098206, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1942586749792099, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19701088964939117, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.006544583011418581, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24052630364894867, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031225604936480522, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7037992477416992, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2955983877182007, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.194245338439941, "metrics/particle/momentum/class_2/accepted/samples": 1281.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5381566286087036, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6070199012756348, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6480916738510132, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.31641891598701477, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28117477893829346, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.3401665687561035, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.051516611129045486, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4213614463806152, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.16507580876350403, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.818260192871094, "metrics/particle/momentum/class_2/assigned/samples": 2077.0, "metrics/particle/momentum/class_2/efficiency": 0.43029895424842834, "metrics/particle/momentum/class_2/event/energy_bias": -14.552803039550781, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.20769691467285, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.427247047424316, "metrics/particle/momentum/class_2/event/px_bias": 0.3623368442058563, "metrics/particle/momentum/class_2/event/py_bias": 0.2362661361694336, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.976688385009766, "metrics/particle/momentum/class_2/event/pz_bias": -1.8296257257461548, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.788911819458008, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022755751386284828, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16053389012813568, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1607341468334198, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.011873865500092506, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2012532651424408, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022278481628745794, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3320457935333252, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.012986955232918262, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2573931217193604, "metrics/particle/momentum/class_3/accepted/samples": 9875.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4240407943725586, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5699687004089355, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.398120164871216, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2339658737182617, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2198782116174698, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.32612419128418, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011744236573576927, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7855983972549438, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.037641219794750214, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4897916316986084, "metrics/particle/momentum/class_3/assigned/samples": 13794.0, "metrics/particle/momentum/class_3/efficiency": 0.6493719816207886, "metrics/particle/momentum/class_3/event/energy_bias": -2.3303332328796387, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.936502456665039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.418116092681885, "metrics/particle/momentum/class_3/event/px_bias": 0.3016432821750641, "metrics/particle/momentum/class_3/event/py_bias": -1.099326252937317, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.676128387451172, "metrics/particle/momentum/class_3/event/pz_bias": -1.9562753438949585, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.220033645629883, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029245784506201744, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16719073057174683, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16061632335186005, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04946111515164375, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2058047503232956, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02118644118309021, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.142882227897644, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3922143578529358, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.866274833679199, "metrics/particle/momentum/class_4/accepted/samples": 236.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3708098530769348, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.039869546890259, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.689548134803772, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5350134372711182, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22121211886405945, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.22434139251709, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.021212121471762657, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2980104684829712, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.42065832018852234, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.712231159210205, "metrics/particle/momentum/class_4/assigned/samples": 330.0, "metrics/particle/momentum/class_4/efficiency": 0.6483516693115234, "metrics/particle/momentum/class_4/event/energy_bias": -1.0142558813095093, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.329951047897339, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2728376388549805, "metrics/particle/momentum/class_4/event/px_bias": -0.28520917892456055, "metrics/particle/momentum/class_4/event/py_bias": 0.11383572965860367, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.015693187713623, "metrics/particle/momentum/class_4/event/pz_bias": -0.07931971549987793, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.777988433837891, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.024545451626181602, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1810818314552307, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18626490235328674, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.002556913997977972, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22818949818611145, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03125, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7688580751419067, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.5173882246017456, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5624337196350098, "metrics/particle/momentum/class_5/accepted/samples": 128.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08200974017381668, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4057799279689789, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.41552233695983887, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2253824770450592, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15131579339504242, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.7946161031723022, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09210526198148727, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.09464430809021, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6457364559173584, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.602323532104492, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.805031418800354, "metrics/particle/momentum/class_5/event/energy_bias": -0.08427095413208008, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.06787109375, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.272312879562378, "metrics/particle/momentum/class_5/event/px_bias": 0.11588181555271149, "metrics/particle/momentum/class_5/event/py_bias": 0.001624874770641327, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6852701902389526, "metrics/particle/momentum/class_5/event/pz_bias": -0.11160553246736526, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.2793192863464355, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4457029700279236, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4488908052444458, "metrics/set_slot/angular_recall_all_dr0p10": 0.9178763031959534, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7530806660652161, "metrics/set_slot/soft_count_bias_mean": -1.864003300666809}, "step_train_time": 5.5466790199279785, "step_valid_time": 46632.585571050644, "step_total_time": 46638.13225030899, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0173544069006686, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16961278551593773, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2837, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2780, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372315824248431, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9378512396694215, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9375413086582948, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06214876033057851, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.918401057152296, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9190082644628099, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9187045604758758, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372315824248431, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029516256754962, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1648660711791622, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2118, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2106, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9363395225464191, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9396628216503993, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9379982285208148, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06033717834960071, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9310344827586207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9343389529724934, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9326837909654562, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9363395225464191, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107800250918153, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17089263820909184, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1160, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9114906832298136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9258675078864353, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.918622848200313, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07413249211356467, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9006211180124224, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9148264984227129, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9076682316118936, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9114906832298136} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_16000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..aaf1c88f91dea4eb67e0944c140f44b3fe3ecaa0 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.7580064430547659, "train": {"Classification_binary": 0.21961799263954163, "Cardinality": 0.16304652392864227, "Classification": 0.1094554141163826, "Regression_pt": 0.14941756427288055, "Regression_eta": 0.0021322935353964567, "Regression_sin_phi": 0.0010592847829684615, "Regression_cos_phi": 0.0012689242139458656, "Regression_energy": 0.1476995050907135, "HitGrouping": 0.17341870069503784, "Auxiliary": 0.24322953820228577, "Total": 1.2559304237365723, "Total_unweighted": 0.6306509786518291}, "train_group": {}, "valid": {"Classification_binary": 0.2041839361190796, "Cardinality": 0.15628538529078165, "Classification": 0.09850257635116577, "Regression_pt": 0.15610963106155396, "Regression_eta": 0.002083118228862683, "Regression_sin_phi": 0.001095488512267669, "Regression_cos_phi": 0.0010693803584824006, "Regression_energy": 0.1533803641796112, "HitGrouping": 0.17142983277638754, "Auxiliary": 0.23582561810811362, "Total": 1.2106293042500813, "Total_unweighted": 0.6164244948110232, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8340578079223633, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5238731503486633, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33541133999824524, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5274309515953064, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5158112645149231, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3769717514514923, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7813666462898254, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.48622873425483704, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34937310218811035, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.49243590235710144, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4857197105884552, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4615384638309479, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8488420248031616, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.514783501625061, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3155149221420288, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5167970061302185, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5019206404685974, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.34763970971107483, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8380321264266968, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5151472687721252, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32495442032814026, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5180338025093079, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5049373507499695, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3672948181629181, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026031259447336197, "metrics/hit_grouping/calorimeter/background_recall": 0.6386882066726685, "metrics/hit_grouping/calorimeter/completeness": 0.5873339772224426, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5959083437919617, "metrics/hit_grouping/calorimeter/energy_purity": 0.5889943838119507, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.38862088322639465, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8611093163490295, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5313618779182434, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25661391019821167, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5376330614089966, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.518496036529541, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.31012243032455444, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.838755190372467, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5366550087928772, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.33721494674682617, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5380215048789978, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5304404497146606, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3734837472438812, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7884323000907898, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4584132134914398, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45683225989341736, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.45571961998939514, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.44966164231300354, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4818606972694397, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8176329135894775, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.537939190864563, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.38604795932769775, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5433831214904785, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5440865159034729, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.42055433988571167, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8067246079444885, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6323003172874451, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5070643424987793, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.628307580947876, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6243388652801514, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.418628990650177, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8639690279960632, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4756532907485962, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2359716147184372, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.47667089104652405, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4475706219673157, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.30346858501434326, "metrics/hit_grouping/calorimeter/purity": 0.5788272023200989, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8570873141288757, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09866780042648315, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11692307889461517, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0975915715098381, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05559481680393219, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8439143300056458, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2877698242664337, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12998859584331512, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2853056788444519, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2587558627128601, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24629418551921844, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8372877836227417, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5371019840240479, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3406020402908325, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5403808355331421, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5286741256713867, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.38129591941833496, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.72452712059021, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44067543745040894, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.483149915933609, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.500914454460144, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5980274677276611, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7920511364936829, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3527762293815613, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5320773124694824, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5344513654708862, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4531688094139099, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6556097865104675, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44695597887039185, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4009888470172882, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.42658206820487976, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6941009759902954, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.688883364200592, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43703949451446533, "metrics/hit_grouping/tracker/all/object_completeness": 0.4379977285861969, "metrics/hit_grouping/tracker/all/object_purity": 0.45916107296943665, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6434441804885864, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005081483628600836, "metrics/hit_grouping/tracker/background_recall": 0.03932950273156166, "metrics/hit_grouping/tracker/completeness": 0.45704784989356995, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4937848150730133, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7181628346443176, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4225057065486908, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5137042999267578, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5260458588600159, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6077448725700378, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.665002167224884, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4443175196647644, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4028978943824768, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44235658645629883, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6834189891815186, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6492609977722168, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.46156957745552063, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30723321437835693, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.32854071259498596, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6802184581756592, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6924381852149963, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.450754851102829, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4389056861400604, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4729938507080078, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6376707553863525, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7061220407485962, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45872169733047485, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4337479770183563, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.46431848406791687, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6151797771453857, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6812213659286499, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.41535940766334534, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4375622570514679, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.44023385643959045, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6559354066848755, "metrics/hit_grouping/tracker/purity": 0.4357263147830963, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8893041610717773, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03518679365515709, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.09586931765079498, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.10196933150291443, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08514335006475449, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6639018654823303, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24468636512756348, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2735612988471985, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.28922298550605774, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5878693461418152, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6644941568374634, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5135470032691956, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5032110214233398, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5270476937294006, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7256360054016113, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5426977872848511, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6220901012420654, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6968206763267517, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6191060543060303, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05697039142251015, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08830127120018005, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.198257446289062, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3019261658191681, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17148631811141968, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8434309363365173, "metrics/particle/class_1/pid_accuracy": 0.9591104388237, "metrics/particle/class_2/efficiency": 0.4690374732017517, "metrics/particle/class_2/pid_accuracy": 0.6682611703872681, "metrics/particle/class_3/efficiency": 0.7251410484313965, "metrics/particle/class_3/pid_accuracy": 0.9566876888275146, "metrics/particle/class_4/efficiency": 0.6814244389533997, "metrics/particle/class_4/pid_accuracy": 0.3968926668167114, "metrics/particle/class_5/efficiency": 0.8999999761581421, "metrics/particle/class_5/pid_accuracy": 0.5, "metrics/particle/count/bias_mean": -1.06640625, "metrics/particle/count/mae": 4.141927242279053, "metrics/particle/count/prediction_mean": 68.98958587646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.08016607910394669, "metrics/particle/event/energy_response_mean": 0.9558927416801453, "metrics/particle/event/met_abs_error": 8.402321815490723, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0666917935013771, "metrics/particle/event/scalar_pt_response_mean": 0.9687312245368958, "metrics/particle/event/vector_pt_closure": 0.06390582025051117, "metrics/particle/matched/delta_eta_abs_mean": 0.01823812909424305, "metrics/particle/matched/delta_phi_abs_mean": 0.01675662398338318, "metrics/particle/matched/delta_r_mean": 0.027675751596689224, "metrics/particle/matched/energy_relative_abs_mean": 0.1521930694580078, "metrics/particle/matched/pid_accuracy": 0.9358420968055725, "metrics/particle/matched/pt_relative_abs_mean": 0.15212146937847137, "metrics/particle/matching/duplicate_fraction": 0.07583421468734741, "metrics/particle/matching/efficiency": 0.7598739862442017, "metrics/particle/matching/f1": 0.7657018303871155, "metrics/particle/matching/purity": 0.7716197371482849, "metrics/particle/matching/target_coverage_dr0p05": 0.6685500741004944, "metrics/particle/matching/target_coverage_dr0p10": 0.804230272769928, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5621433854103088, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8147315382957458, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026010578498244286, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1511927843093872, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15207727253437042, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02689138799905777, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1909729242324829, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010049019940197468, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8234540224075317, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.13623596727848053, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2635703086853027, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8160.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3433955907821655, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3181689977645874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7477531433105469, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5307902097702026, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15685328841209412, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 34.11052703857422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0210424717515707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.247015118598938, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.16744579374790192, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.872420072555542, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10360.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7461594939231873, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -5.039247512817383, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.508216857910156, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.772028923034668, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.593372642993927, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.03234793618321419, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.099066734313965, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.8986525535583496, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.271564483642578, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02421446703374386, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1498849242925644, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1493011713027954, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.025308281183242798, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18719126284122467, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0295670535415411, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5960874557495117, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.3259370028972626, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.340646743774414, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1894.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5543617010116577, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0520449876785278, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3535592555999756, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1401407718658447, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19522692263126373, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.567835807800293, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06181533634662628, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.569394111633301, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.3100140690803528, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.911057472229004, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2556.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6269447207450867, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.85487699508667, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.862255096435547, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.490939974784851, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08882634341716766, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.22967109084129333, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.679337501525879, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.750460147857666, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.86363410949707, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028497450053691864, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15209276974201202, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15185511112213135, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.019701611250638962, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19128796458244324, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001254705130122602, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2756209373474121, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03519972786307335, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8521329164505005, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16737.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2582438290119171, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2518744468688965, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0156468152999878, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8066896796226501, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14886347949504852, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.764163970947266, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004697954747825861, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.481202632188797, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.052549950778484344, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9129287004470825, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21073.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7807528972625732, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.4655631482601166, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.412792205810547, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.738421440124512, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1590087413787842, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1610302925109863, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.768608570098877, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7703310251235962, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.559518814086914, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027437210083007812, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15166257321834564, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15174223482608795, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.02228783816099167, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19090518355369568, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005934828892350197, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5358307957649231, "metrics/particle/momentum/all/accepted/pz_bias": 0.08652712404727936, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.829264760017395, "metrics/particle/momentum/all/accepted/samples": 26791.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3064667582511902, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2570538520812988, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.339408278465271, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1276754140853882, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1547853648662567, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.254596710205078, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013975109905004501, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8716593384742737, "metrics/particle/momentum/all/assigned/pz_bias": 0.10693222284317017, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2297487258911133, "metrics/particle/momentum/all/assigned/samples": 33989.0, "metrics/particle/momentum/all/efficiency": 0.7569361925125122, "metrics/particle/momentum/all/event/energy_bias": -12.359686851501465, "metrics/particle/momentum/all/event/p3_error_mean": 23.688262939453125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.143774032592773, "metrics/particle/momentum/all/event/px_bias": 1.84120774269104, "metrics/particle/momentum/all/event/py_bias": -1.3583534955978394, "metrics/particle/momentum/all/event/pz_abs_mean": 18.290416717529297, "metrics/particle/momentum/all/event/pz_bias": 6.4194440841674805, "metrics/particle/momentum/all/event/pz_residual_rms": 26.09710121154785, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030672073364257812, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13975122570991516, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14101630449295044, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0232273880392313, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17903576791286469, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006407974287867546, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5830206274986267, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.11340714991092682, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8689433336257935, "metrics/particle/momentum/class_1/accepted/samples": 14045.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18550054728984833, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4336357116699219, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4365784525871277, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23559033870697021, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09435577690601349, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.7049927711486816, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011435210704803467, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.793505072593689, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.13931123912334442, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7771222591400146, "metrics/particle/momentum/class_1/assigned/samples": 16353.0, "metrics/particle/momentum/class_1/efficiency": 0.8416731357574463, "metrics/particle/momentum/class_1/event/energy_bias": 1.76552414894104, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.557777404785156, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.883069038391113, "metrics/particle/momentum/class_1/event/px_bias": 2.8625595569610596, "metrics/particle/momentum/class_1/event/py_bias": -1.688666582107544, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.093391418457031, "metrics/particle/momentum/class_1/event/pz_bias": 4.673746585845947, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.088497161865234, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03149742633104324, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18965056538581848, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1901855617761612, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022210119292140007, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23032455146312714, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.017830608412623405, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5454778671264648, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.2372969388961792, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8215537071228027, "metrics/particle/momentum/class_2/accepted/samples": 1346.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.697910487651825, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6608958840370178, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8853089213371277, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4873529076576233, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3242236077785492, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.74117374420166, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05383022874593735, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.544020175933838, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.3084124028682709, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.465425968170166, "metrics/particle/momentum/class_2/assigned/samples": 2415.0, "metrics/particle/momentum/class_2/efficiency": 0.45213302969932556, "metrics/particle/momentum/class_2/event/energy_bias": -13.753487586975098, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.016010284423828, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.676135063171387, "metrics/particle/momentum/class_2/event/px_bias": -0.5959761142730713, "metrics/particle/momentum/class_2/event/py_bias": -0.13507941365242004, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.67529582977295, "metrics/particle/momentum/class_2/event/pz_bias": 0.6598009467124939, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.854516983032227, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022826388478279114, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16164036095142365, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16008517146110535, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.021737342700362206, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19931572675704956, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0027267769910395145, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3139285743236542, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.016769448295235634, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2240430116653442, "metrics/particle/momentum/class_3/accepted/samples": 11002.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3777129054069519, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.2672626972198486, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4132487773895264, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2190685272216797, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1935681253671646, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 30.415388107299805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009382649324834347, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6542401909828186, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.020083732903003693, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.8317208290100098, "metrics/particle/momentum/class_3/assigned/samples": 14708.0, "metrics/particle/momentum/class_3/efficiency": 0.723482608795166, "metrics/particle/momentum/class_3/event/energy_bias": 1.4575446844100952, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.231128692626953, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.042824745178223, "metrics/particle/momentum/class_3/event/px_bias": -0.7687897682189941, "metrics/particle/momentum/class_3/event/py_bias": 0.49878963828086853, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.102367401123047, "metrics/particle/momentum/class_3/event/pz_bias": 0.32533520460128784, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.263582229614258, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028941994532942772, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16701920330524445, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16495616734027863, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.043748244643211365, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20502717792987823, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01984127052128315, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2672343254089355, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.7334129810333252, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.8681771755218506, "metrics/particle/momentum/class_4/accepted/samples": 252.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3738768398761749, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8512613773345947, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0208067893981934, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.880969762802124, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2240896373987198, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.69328498840332, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016806723549962044, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3214949369430542, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.6446774005889893, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6297175884246826, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.692307710647583, "metrics/particle/momentum/class_4/event/energy_bias": -0.42482587695121765, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.62919020652771, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.345780611038208, "metrics/particle/momentum/class_4/event/px_bias": 0.2731245458126068, "metrics/particle/momentum/class_4/event/py_bias": -0.22718483209609985, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.2989675998687744, "metrics/particle/momentum/class_4/event/pz_bias": 0.43789589405059814, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.255037307739258, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02367265895009041, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16890785098075867, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17764511704444885, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08808451890945435, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22091494500637054, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06849315017461777, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.147392988204956, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.250858873128891, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.0335373878479, "metrics/particle/momentum/class_5/accepted/samples": 146.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.055634889751672745, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1980576366186142, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20684711635112762, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.06778398901224136, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.27462872862815857, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.644193410873413, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5513179302215576, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.567099094390869, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.9182389974594116, "metrics/particle/momentum/class_5/event/energy_bias": -1.4044431447982788, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.635935068130493, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9754432439804077, "metrics/particle/momentum/class_5/event/px_bias": 0.07028956711292267, "metrics/particle/momentum/class_5/event/py_bias": 0.19378772377967834, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4132771492004395, "metrics/particle/momentum/class_5/event/pz_bias": 0.32266443967819214, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.489505767822266, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42114123702049255, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4253152310848236, "metrics/set_slot/angular_recall_all_dr0p10": 0.9312306046485901, "metrics/set_slot/angular_recall_selected_dr0p10": 0.791433572769165, "metrics/set_slot/soft_count_bias_mean": 2.1298394203186035}, "step_train_time": 6.137321472167969, "step_valid_time": 49724.08131790161, "step_total_time": 49730.2186396122, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.996511547929413, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17756182548740895, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3007, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2855, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2823, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9431780640898579, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9494512803458597, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.946304275770633, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05054871965414034, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9326065411298315, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9388094446291986, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9356977129598939, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9431780640898579, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9582632116825455, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17112863712177728, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2227, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2108, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2100, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9319186560565871, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9465648854961832, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9391846736466919, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05343511450381679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9283819628647215, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9429726088908846, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9356204054355091, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9319186560565871, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9627992080493152, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16722693208524564, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1253, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1164, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1157, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9037267080745341, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9289704708699122, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9161747343565525, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0710295291300878, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8982919254658385, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9233838786911412, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9106650924832743, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9037267080745341} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_17000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..2dee1413bfb71ee23abd564b608285d96d4fd45d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8053818457456888, "train": {"Classification_binary": 0.2107614427804947, "Cardinality": 0.12275054305791855, "Classification": 0.0974699854850769, "Regression_pt": 0.15255646407604218, "Regression_eta": 0.0023391940630972385, "Regression_sin_phi": 0.001333626569248736, "Regression_cos_phi": 0.001287157996557653, "Regression_energy": 0.15076670050621033, "HitGrouping": 0.17216454446315765, "Auxiliary": 0.23161499202251434, "Total": 1.217552900314331, "Total_unweighted": 0.6165145714767277}, "train_group": {}, "valid": {"Classification_binary": 0.1981403430302938, "Cardinality": 0.13613110780715942, "Classification": 0.09281531969706218, "Regression_pt": 0.15133623282114664, "Regression_eta": 0.00218581889445583, "Regression_sin_phi": 0.0011519613520552714, "Regression_cos_phi": 0.0011585163107762735, "Regression_energy": 0.15072144071261087, "HitGrouping": 0.17063001791636148, "Auxiliary": 0.2284980614980062, "Total": 1.1827976703643799, "Total_unweighted": 0.5975096328184009, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8378875851631165, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5286708474159241, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33125531673431396, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5329434871673584, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5223662853240967, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.36667612195014954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7919272184371948, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5034195780754089, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33649611473083496, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5105516314506531, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.49982377886772156, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4293459951877594, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8531603217124939, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5215855240821838, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.31525668501853943, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.524103581905365, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.508131206035614, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33906620740890503, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8427533507347107, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5222360491752625, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32231003046035767, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5257326364517212, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5119668245315552, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3560486435890198, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02616967260837555, "metrics/hit_grouping/calorimeter/background_recall": 0.6404957175254822, "metrics/hit_grouping/calorimeter/completeness": 0.5860371589660645, "metrics/hit_grouping/calorimeter/energy_completeness": 0.593349814414978, "metrics/hit_grouping/calorimeter/energy_purity": 0.5865498781204224, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37655436992645264, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8614394068717957, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5390658378601074, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25739026069641113, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5458521246910095, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5280044674873352, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3070170283317566, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8442733883857727, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.544274091720581, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3361232280731201, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5471884608268738, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5381983518600464, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.35953420400619507, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8016169667243958, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.46364733576774597, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4453292787075043, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.460787296295166, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.45068472623825073, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4562002420425415, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8247430920600891, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5471318364143372, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3725942075252533, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5531308650970459, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5541945695877075, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4090937376022339, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8214440941810608, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6198284029960632, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5034013390541077, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6160909533500671, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6005332469940186, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3929879665374756, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8648422956466675, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.48298338055610657, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24507834017276764, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4847153425216675, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.45523613691329956, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29429489374160767, "metrics/hit_grouping/calorimeter/purity": 0.5776312947273254, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8520280718803406, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11435579508543015, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1230769231915474, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1131959781050682, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07710639387369156, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03076923079788685, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.841545820236206, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2894497215747833, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1351197212934494, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28986525535583496, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2670995593070984, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23375143110752106, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8426250219345093, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.544318675994873, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3373229205608368, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5480916500091553, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5353192687034607, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3700157403945923, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.721155047416687, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43066346645355225, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4862368404865265, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5149270296096802, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6052002310752869, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7944759130477905, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3241727352142334, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5381729006767273, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5519941449165344, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4425126314163208, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6634839773178101, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4394738972187042, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.40354132652282715, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.42435890436172485, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6863825917243958, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6927439570426941, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42697903513908386, "metrics/hit_grouping/tracker/all/object_completeness": 0.4410175383090973, "metrics/hit_grouping/tracker/all/object_purity": 0.4637342691421509, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6401851773262024, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.003885840531438589, "metrics/hit_grouping/tracker/background_recall": 0.02949712797999382, "metrics/hit_grouping/tracker/completeness": 0.45818620920181274, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.48390236496925354, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7202466726303101, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4161275625228882, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5184676051139832, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5347235202789307, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6123006939888, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6651720404624939, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4420331120491028, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4049241542816162, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4444720149040222, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6807538270950317, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6609868407249451, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43507280945777893, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30743837356567383, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3266141414642334, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6589806079864502, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6964895725250244, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4353882074356079, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43965330719947815, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4759424030780792, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6304816603660583, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7250437140464783, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4693741798400879, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.43244311213493347, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43539246916770935, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5792276859283447, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6821850538253784, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40856942534446716, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4443025290966034, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4528138041496277, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6635448336601257, "metrics/hit_grouping/tracker/purity": 0.4362872540950775, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8896491527557373, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.044743701815605164, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.10129246860742569, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.10367154330015182, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6775001287460327, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2545360326766968, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2975307106971741, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3039703965187073, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5635043978691101, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6677235960960388, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4985538125038147, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.50351881980896, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5308524966239929, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.725399911403656, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5287894606590271, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6161930561065674, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.688842236995697, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6247054934501648, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.04306209087371826, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.093900665640831, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.9736385345459, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29476597905158997, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16432614624500275, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.83610999584198, "metrics/particle/class_1/pid_accuracy": 0.9585394263267517, "metrics/particle/class_2/efficiency": 0.4766659140586853, "metrics/particle/class_2/pid_accuracy": 0.6693339347839355, "metrics/particle/class_3/efficiency": 0.7008559703826904, "metrics/particle/class_3/pid_accuracy": 0.961523175239563, "metrics/particle/class_4/efficiency": 0.6920115351676941, "metrics/particle/class_4/pid_accuracy": 0.452016681432724, "metrics/particle/class_5/efficiency": 0.878000020980835, "metrics/particle/class_5/pid_accuracy": 0.6104783415794373, "metrics/particle/count/bias_mean": -1.484375, "metrics/particle/count/mae": 4.061197757720947, "metrics/particle/count/prediction_mean": 68.57161712646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.0816570520401001, "metrics/particle/event/energy_response_mean": 1.0175079107284546, "metrics/particle/event/met_abs_error": 8.08932113647461, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06919126212596893, "metrics/particle/event/scalar_pt_response_mean": 1.0247806310653687, "metrics/particle/event/vector_pt_closure": 0.06386800110340118, "metrics/particle/matched/delta_eta_abs_mean": 0.017042124643921852, "metrics/particle/matched/delta_phi_abs_mean": 0.016903160139918327, "metrics/particle/matched/delta_r_mean": 0.026898566633462906, "metrics/particle/matched/energy_relative_abs_mean": 0.1498044729232788, "metrics/particle/matched/pid_accuracy": 0.9380204677581787, "metrics/particle/matched/pt_relative_abs_mean": 0.14843620359897614, "metrics/particle/matching/duplicate_fraction": 0.0719100683927536, "metrics/particle/matching/efficiency": 0.7465475797653198, "metrics/particle/matching/f1": 0.7545413374900818, "metrics/particle/matching/purity": 0.7627081871032715, "metrics/particle/matching/target_coverage_dr0p05": 0.6697582006454468, "metrics/particle/matching/target_coverage_dr0p10": 0.7986636161804199, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5653309226036072, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8032730221748352, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025644056499004364, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14894725382328033, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14653334021568298, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012256245128810406, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18655963242053986, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007551373913884163, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7576977610588074, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.12004483491182327, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9879436492919922, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8078.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33418574929237366, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2328243255615234, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4249383211135864, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2548003196716309, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16573305428028107, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.133279800415039, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019702529534697533, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2175605297088623, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1271657794713974, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.448917865753174, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10354.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7386612892150879, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.1235456466674805, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.941397666931152, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.92214822769165, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1262110471725464, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.16848912835121155, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.445521354675293, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -5.069949150085449, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.909561157226562, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024459173902869225, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1522047072649002, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1498991698026657, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.030012132599949837, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1888420581817627, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027993779629468918, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4830503463745117, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.23756948113441467, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6304190158843994, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1929.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.506608247756958, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9034121632575989, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1121466159820557, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9520151615142822, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.198464497923851, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 13.610445976257324, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06180422380566597, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.522584915161133, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.35556986927986145, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.8813652992248535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2605.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6385303139686584, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.525852680206299, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.00329875946045, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4027893543243408, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0747494250535965, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08788146078586578, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.808728218078613, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.434436559677124, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.432889938354492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027479249984025955, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1492924839258194, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14812548458576202, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008159750141203403, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18814365565776825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008544400334358215, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26429781317710876, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.049222081899642944, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7925114631652832, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16385.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2814907431602478, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7177691459655762, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3088654279708862, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.126473069190979, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15907682478427887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.923832893371582, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007057365030050278, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5504350662231445, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.07237198948860168, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.5327985286712646, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20971.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7643327116966248, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.49679708480835, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.965118408203125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.13935661315918, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8077347874641418, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.8787367939949036, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.041087627410889, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.740585207939148, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.226116180419922, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02669679932296276, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14939968287944794, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1477678120136261, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01101079210639, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18771150708198547, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004887844901531935, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5043954849243164, "metrics/particle/momentum/all/accepted/pz_bias": -0.08466571569442749, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6008901596069336, "metrics/particle/momentum/all/accepted/samples": 26392.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.31485459208488464, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5072616338729858, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.405958652496338, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.22901451587677, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1641320437192917, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.11003589630127, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01511936355382204, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9054267406463623, "metrics/particle/momentum/all/assigned/pz_bias": -0.11083545535802841, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.351175308227539, "metrics/particle/momentum/all/assigned/samples": 33930.0, "metrics/particle/momentum/all/efficiency": 0.7456631064414978, "metrics/particle/momentum/all/event/energy_bias": 6.094490051269531, "metrics/particle/momentum/all/event/p3_error_mean": 23.09614372253418, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.53149700164795, "metrics/particle/momentum/all/event/px_bias": -1.008695125579834, "metrics/particle/momentum/all/event/py_bias": 0.9593443274497986, "metrics/particle/momentum/all/event/pz_abs_mean": 17.370641708374023, "metrics/particle/momentum/all/event/pz_bias": -8.24497127532959, "metrics/particle/momentum/all/event/pz_residual_rms": 26.81098175048828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029316017404198647, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13503386080265045, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13440141081809998, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.015006005764007568, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1717633306980133, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004147597122937441, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5214278697967529, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.10373033583164215, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5344098806381226, "metrics/particle/momentum/class_1/accepted/samples": 13984.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.19679275155067444, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.47718146443367004, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5191834568977356, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3325350284576416, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09240498393774033, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.16494607925415, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009209799580276012, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7171086072921753, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.09190527349710464, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2908575534820557, "metrics/particle/momentum/class_1/assigned/samples": 16287.0, "metrics/particle/momentum/class_1/efficiency": 0.8380176424980164, "metrics/particle/momentum/class_1/event/energy_bias": 6.790313720703125, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.9289493560791, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.545388221740723, "metrics/particle/momentum/class_1/event/px_bias": -0.09757031500339508, "metrics/particle/momentum/class_1/event/py_bias": 0.37685221433639526, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.621805191040039, "metrics/particle/momentum/class_1/event/pz_bias": -4.632464408874512, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.8426456451416, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031835127621889114, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1925021857023239, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19182123243808746, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.008618192747235298, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23325419425964355, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026599569246172905, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5438835620880127, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10248608142137527, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.600825309753418, "metrics/particle/momentum/class_2/accepted/samples": 1391.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6647151112556458, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8566683530807495, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1799904108047485, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8309105038642883, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.31866776943206787, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 10.386886596679688, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06332236528396606, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.691986083984375, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2338324338197708, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.3535661697387695, "metrics/particle/momentum/class_2/assigned/samples": 2432.0, "metrics/particle/momentum/class_2/efficiency": 0.46724891662597656, "metrics/particle/momentum/class_2/event/energy_bias": -6.332571029663086, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.64575958251953, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.90255355834961, "metrics/particle/momentum/class_2/event/px_bias": -0.710873544216156, "metrics/particle/momentum/class_2/event/py_bias": 0.12307542562484741, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.04545783996582, "metrics/particle/momentum/class_2/event/pz_bias": -2.670590877532959, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.78898048400879, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02257932722568512, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16263744235038757, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15942999720573425, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.045803628861904144, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2003958374261856, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022605254780501127, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3191532492637634, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05652041733264923, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1400917768478394, "metrics/particle/momentum/class_3/accepted/samples": 10617.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.388468474149704, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.727431297302246, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4014089107513428, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2638697624206543, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2176634669303894, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.48093032836914, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01313193142414093, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7981734871864319, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.11139339953660965, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.598055839538574, "metrics/particle/momentum/class_3/assigned/samples": 14697.0, "metrics/particle/momentum/class_3/efficiency": 0.6981652975082397, "metrics/particle/momentum/class_3/event/energy_bias": 6.640639781951904, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.176207542419434, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.233553886413574, "metrics/particle/momentum/class_3/event/px_bias": -0.02652433142066002, "metrics/particle/momentum/class_3/event/py_bias": 0.4174712896347046, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.032962799072266, "metrics/particle/momentum/class_3/event/pz_bias": -1.1322510242462158, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.09950828552246, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028045689687132835, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1489226520061493, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14889444410800934, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.019116966053843498, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1941525638103485, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.019305018708109856, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9038904309272766, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08632519096136093, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2761051654815674, "metrics/particle/momentum/class_4/accepted/samples": 259.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3812595307826996, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.2555530071258545, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.9287407398223877, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7610385417938232, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21288515627384186, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.951507568359375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02240896411240101, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1927158832550049, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2300293892621994, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.5563666820526123, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.7115384340286255, "metrics/particle/momentum/class_4/event/energy_bias": -0.8113579750061035, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.2058603763580322, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0360193252563477, "metrics/particle/momentum/class_4/event/px_bias": 0.1428467482328415, "metrics/particle/momentum/class_4/event/py_bias": -0.025625064969062805, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.09751033782959, "metrics/particle/momentum/class_4/event/pz_bias": 0.11511410772800446, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.87611722946167, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02379772998392582, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15304690599441528, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1586061716079712, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01982611045241356, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19869574904441833, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03546099364757538, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7748911380767822, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.45145562291145325, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.301743984222412, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10085485130548477, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2473178505897522, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2511167824268341, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03761807084083557, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08917197585105896, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6002066731452942, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05095541477203369, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1536080837249756, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.8922088146209717, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.0810322761535645, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.8867924809455872, "metrics/particle/momentum/class_5/event/energy_bias": -0.19253422319889069, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.4677138328552246, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8892011642456055, "metrics/particle/momentum/class_5/event/px_bias": -0.3165736794471741, "metrics/particle/momentum/class_5/event/py_bias": 0.0675705298781395, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2927814722061157, "metrics/particle/momentum/class_5/event/pz_bias": 0.07522162795066833, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.008978366851807, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41906702518463135, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42044711112976074, "metrics/set_slot/angular_recall_all_dr0p10": 0.9254781603813171, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7880043983459473, "metrics/set_slot/soft_count_bias_mean": 0.7858070731163025}, "step_train_time": 6.455380201339722, "step_valid_time": 52859.72120499611, "step_total_time": 52866.176585674286, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0237625903262835, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17330009772147437, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3017, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2856, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2823, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9435084241823588, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9466357308584686, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9450694904037061, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05336426914153132, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9326065411298315, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9356977129598939, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9341495698213104, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9435084241823588, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0202749785162182, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16490224604542814, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2236, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2127, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2121, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9403183023872679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512522361359571, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457536682970209, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048747763864042934, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9376657824933687, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485688729874776, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9430858159181859, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9403183023872679, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013474263667279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1901916758555633, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1166, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9130434782608695, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9274447949526814, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.92018779342723, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07255520504731862, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.90527950310559, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.919558359621451, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9123630672926447, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9130434782608695} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_18000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7cab3bc34f7d4f1b674decf47ba3b2db3a29cc --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.8527572484366117, "train": {"Classification_binary": 0.19164110720157623, "Cardinality": 0.1496310532093048, "Classification": 0.08891624212265015, "Regression_pt": 0.13204137980937958, "Regression_eta": 0.0018711547600105405, "Regression_sin_phi": 0.0010009005200117826, "Regression_cos_phi": 0.0009606224484741688, "Regression_energy": 0.13095933198928833, "HitGrouping": 0.16665621101856232, "Auxiliary": 0.22991931438446045, "Total": 1.1292210817337036, "Total_unweighted": 0.5473907388513908}, "train_group": {}, "valid": {"Classification_binary": 0.19040562709172568, "Cardinality": 0.12695212165514627, "Classification": 0.08666254083315532, "Regression_pt": 0.1420121192932129, "Regression_eta": 0.0018127989023923874, "Regression_sin_phi": 0.001035151770338416, "Regression_cos_phi": 0.0010214131325483322, "Regression_energy": 0.14088099201520285, "HitGrouping": 0.16758714119593301, "Auxiliary": 0.2183844049771627, "Total": 1.109385093053182, "Total_unweighted": 0.5638306430385759, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8353160619735718, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5472783446311951, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.32492679357528687, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5533146858215332, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5497764945030212, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.36639273166656494, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7870357632637024, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5072934627532959, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3192138373851776, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5155670046806335, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.521934449672699, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43781769275665283, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8525750637054443, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5335791110992432, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3059084713459015, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5355938673019409, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5292383432388306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3431463837623596, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.841142475605011, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5356296896934509, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.31322187185287476, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5394999980926514, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5351923108100891, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3591185510158539, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025947319343686104, "metrics/hit_grouping/calorimeter/background_recall": 0.6378985643386841, "metrics/hit_grouping/calorimeter/completeness": 0.6048154234886169, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6147472858428955, "metrics/hit_grouping/calorimeter/energy_purity": 0.6075300574302673, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36818307638168335, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8616294264793396, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5495598912239075, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24992534518241882, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5573883056640625, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5473783612251282, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.30486711859703064, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8427421450614929, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5599189400672913, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.32678312063217163, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5616552829742432, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5578855276107788, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3685104250907898, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7961119413375854, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4808320999145508, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4330678880214691, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4785483181476593, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4857497215270996, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.46416381001472473, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8228466510772705, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5624196529388428, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.36779817938804626, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5689464211463928, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5814124345779419, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.40890687704086304, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8122538328170776, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.647686243057251, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.47671377658843994, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6419378519058228, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6404735445976257, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40607011318206787, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8645085692405701, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.49248966574668884, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23362796008586884, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.49473488330841064, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.47203052043914795, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29958483576774597, "metrics/hit_grouping/calorimeter/purity": 0.5960041284561157, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8400610089302063, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1386120319366455, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13692307472229004, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.13703662157058716, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08862141519784927, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8424108028411865, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29799818992614746, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1220068410038948, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2964193522930145, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.27835115790367126, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24116306006908417, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8410926461219788, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5577593445777893, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3279774487018585, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5620591044425964, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.559482991695404, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.37273740768432617, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7296149730682373, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41512253880500793, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49908480048179626, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5332654118537903, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5974297523498535, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7967832088470459, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3180033564567566, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5532619953155518, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.567236065864563, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43129557371139526, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6638098359107971, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4350633919239044, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.41193822026252747, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45026639103889465, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.681657075881958, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.695807695388794, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41890233755111694, "metrics/hit_grouping/tracker/all/object_completeness": 0.45138487219810486, "metrics/hit_grouping/tracker/all/object_purity": 0.4863511323928833, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6339504718780518, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0046941195614635944, "metrics/hit_grouping/tracker/background_recall": 0.03624948114156723, "metrics/hit_grouping/tracker/completeness": 0.46954086422920227, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46756085753440857, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.728787899017334, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40792709589004517, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5319363474845886, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.552777886390686, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5971754193305969, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6685760021209717, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42832666635513306, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.41089656949043274, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4663555324077606, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6758043169975281, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6515300869941711, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4332524240016937, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31559062004089355, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3601383566856384, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6711165308952332, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6972361207008362, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42316678166389465, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44738805294036865, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.501558244228363, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6264378428459167, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7140877842903137, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4527297019958496, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.44388341903686523, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.477039635181427, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5938748121261597, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6907327175140381, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4073987305164337, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4579107463359833, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4681776762008667, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6507843732833862, "metrics/hit_grouping/tracker/purity": 0.4474622905254364, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8876147866249084, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04126846045255661, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.10467503219842911, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.10843963176012039, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07819287478923798, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6737985014915466, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25920164585113525, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.31487706303596497, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3317455053329468, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5692068338394165, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6722504496574402, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4884009063243866, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5140405893325806, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5553072690963745, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7168408036231995, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5303587913513184, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6675401329994202, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7522167563438416, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6382156610488892, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.044631462544202805, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.10741090774536133, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.08053970336914, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29669389128685, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16625408828258514, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8571626543998718, "metrics/particle/class_1/pid_accuracy": 0.9632741808891296, "metrics/particle/class_2/efficiency": 0.46746692061424255, "metrics/particle/class_2/pid_accuracy": 0.6815454959869385, "metrics/particle/class_3/efficiency": 0.7227597236633301, "metrics/particle/class_3/pid_accuracy": 0.9651528596878052, "metrics/particle/class_4/efficiency": 0.6881616711616516, "metrics/particle/class_4/pid_accuracy": 0.43916085362434387, "metrics/particle/class_5/efficiency": 0.9020000100135803, "metrics/particle/class_5/pid_accuracy": 0.5898004174232483, "metrics/particle/count/bias_mean": -2.1640625, "metrics/particle/count/mae": 4.203125, "metrics/particle/count/prediction_mean": 67.89192962646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07633420079946518, "metrics/particle/event/energy_response_mean": 0.9593567848205566, "metrics/particle/event/met_abs_error": 7.645480632781982, "metrics/particle/event/scalar_pt_relative_abs_error": 0.060238663107156754, "metrics/particle/event/scalar_pt_response_mean": 0.9836020469665527, "metrics/particle/event/vector_pt_closure": 0.060660600662231445, "metrics/particle/matched/delta_eta_abs_mean": 0.01674620993435383, "metrics/particle/matched/delta_phi_abs_mean": 0.01666112057864666, "metrics/particle/matched/delta_r_mean": 0.026498746126890182, "metrics/particle/matched/energy_relative_abs_mean": 0.14543108642101288, "metrics/particle/matched/pid_accuracy": 0.9431895613670349, "metrics/particle/matched/pt_relative_abs_mean": 0.14618730545043945, "metrics/particle/matching/duplicate_fraction": 0.07015592604875565, "metrics/particle/matching/efficiency": 0.7652361392974854, "metrics/particle/matching/f1": 0.7772408127784729, "metrics/particle/matching/purity": 0.7896281480789185, "metrics/particle/matching/target_coverage_dr0p05": 0.6916900277137756, "metrics/particle/matching/target_coverage_dr0p10": 0.815140426158905, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5797074437141418, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8122409582138062, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025630230084061623, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14482051134109497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14572495222091675, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012814821675419807, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18528908491134644, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007897933945059776, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7377477884292603, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07019025832414627, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.946942925453186, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8230.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29046085476875305, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9850928783416748, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1292996406555176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9628511071205139, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14696110785007477, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.041041374206543, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015634160488843918, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0963551998138428, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.022414028644561768, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.127471923828125, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10234.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7525603771209717, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.890537977218628, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.371150970458984, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.338515281677246, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2977551519870758, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1992257535457611, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.958540439605713, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.228358507156372, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.633060455322266, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024289026856422424, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14673875272274017, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1441936492919922, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007717794273048639, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18381623923778534, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024109015241265297, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4124393463134766, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.07374481111764908, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.642702579498291, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1908.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5036908984184265, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1593941450119019, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.6588516235351562, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4904773235321045, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18454763293266296, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.974157333374023, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.057646118104457855, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4712300300598145, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.030320828780531883, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.417569637298584, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2498.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6315789222717285, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.219782829284668, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.707390785217285, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4627084732055664, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17035029828548431, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10281167924404144, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.540924072265625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.814807116985321, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.90264129638672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02669135108590126, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14467088878154755, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14581666886806488, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017864961177110672, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18597368896007538, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009485416230745614, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.25837603211402893, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015542589128017426, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8329973220825195, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16868.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.24223549664020538, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2202849388122559, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9201246500015259, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7490387558937073, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14471986889839172, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.724302291870117, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005110569763928652, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.46623584628105164, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.010719641111791134, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.971872091293335, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20937.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7868638038635254, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.666862964630127, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.350550651550293, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.767023086547852, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.9924455881118774, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1111294031143188, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.6204047203063965, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5078855156898499, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.797912120819092, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0261982511729002, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1448625922203064, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14567406475543976, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01560903713107109, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18561355769634247, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004702658858150244, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.48599886894226074, "metrics/particle/momentum/all/accepted/pz_bias": -0.036308351904153824, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5893582105636597, "metrics/particle/momentum/all/accepted/samples": 27006.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2762921452522278, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1442785263061523, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1127065420150757, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9432313442230225, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14835605025291443, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.40302562713623, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012207075022161007, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8065226674079895, "metrics/particle/momentum/all/assigned/pz_bias": -0.015728527680039406, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0778377056121826, "metrics/particle/momentum/all/assigned/samples": 33669.0, "metrics/particle/momentum/all/efficiency": 0.7630106806755066, "metrics/particle/momentum/all/event/energy_bias": -11.777183532714844, "metrics/particle/momentum/all/event/p3_error_mean": 21.443309783935547, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.879009246826172, "metrics/particle/momentum/all/event/px_bias": 0.5243400931358337, "metrics/particle/momentum/all/event/py_bias": -1.4131667613983154, "metrics/particle/momentum/all/event/pz_abs_mean": 15.755138397216797, "metrics/particle/momentum/all/event/pz_bias": -2.535280466079712, "metrics/particle/momentum/all/event/pz_residual_rms": 23.668787002563477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027770476415753365, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1315804123878479, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.133320152759552, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007903712801635265, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17080780863761902, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003991876263171434, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5020861625671387, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.06029294431209564, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4469550848007202, "metrics/particle/momentum/class_1/accepted/samples": 14279.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1540607064962387, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3792787790298462, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4020252227783203, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24165020883083344, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08264767378568649, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.110517501831055, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008719145320355892, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.67756587266922, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03548431769013405, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.172440528869629, "metrics/particle/momentum/class_1/assigned/samples": 16286.0, "metrics/particle/momentum/class_1/efficiency": 0.8556960225105286, "metrics/particle/momentum/class_1/event/energy_bias": 3.2560272216796875, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.237977981567383, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.452010154724121, "metrics/particle/momentum/class_1/event/px_bias": -0.4559177756309509, "metrics/particle/momentum/class_1/event/py_bias": -0.7551719546318054, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.942721366882324, "metrics/particle/momentum/class_1/event/pz_bias": -4.244109153747559, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.736074447631836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.030544141307473183, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1884046047925949, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1880524754524231, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026150377467274666, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22870196402072906, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025735294446349144, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5700922012329102, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09751878678798676, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.864718198776245, "metrics/particle/momentum/class_2/accepted/samples": 1360.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7114346623420715, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6674712300300598, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8341441750526428, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.44116535782814026, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3074614107608795, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.083301067352295, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.048456259071826935, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5116653442382812, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.12473248690366745, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.732533931732178, "metrics/particle/momentum/class_2/assigned/samples": 2332.0, "metrics/particle/momentum/class_2/efficiency": 0.4568357467651367, "metrics/particle/momentum/class_2/event/energy_bias": -17.164384841918945, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.09819793701172, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.039063453674316, "metrics/particle/momentum/class_2/event/px_bias": 0.3863919675350189, "metrics/particle/momentum/class_2/event/py_bias": -0.6266354918479919, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.31957721710205, "metrics/particle/momentum/class_2/event/pz_bias": 0.8067022562026978, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.23562240600586, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02362102083861828, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1562928408384323, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15606099367141724, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030806617811322212, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19730861485004425, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021889822091907263, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29855915904045105, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0033953923266381025, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1350398063659668, "metrics/particle/momentum/class_3/accepted/samples": 10964.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3429325520992279, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.067596197128296, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9472935199737549, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8031818866729736, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19597195088863373, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.075735092163086, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009692054241895676, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.650907576084137, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.02556142397224903, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.9675495624542236, "metrics/particle/momentum/class_3/assigned/samples": 14548.0, "metrics/particle/momentum/class_3/efficiency": 0.7209837436676025, "metrics/particle/momentum/class_3/event/energy_bias": 3.329680919647217, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.51846981048584, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.521178245544434, "metrics/particle/momentum/class_3/event/px_bias": 0.32781314849853516, "metrics/particle/momentum/class_3/event/py_bias": 0.050193265080451965, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.642519950866699, "metrics/particle/momentum/class_3/event/pz_bias": 0.5589848756790161, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.05405044555664, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02879152074456215, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15866795182228088, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15996433794498444, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.034395087510347366, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19791069626808167, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.019305018708109856, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0503791570663452, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.26497742533683777, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.5598461627960205, "metrics/particle/momentum/class_4/accepted/samples": 259.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3742525279521942, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.956594705581665, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7466039657592773, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5905539989471436, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2103746384382248, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.176514625549316, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020172910764813423, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2521075010299683, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.32676371932029724, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.5551304817199707, "metrics/particle/momentum/class_4/assigned/samples": 347.0, "metrics/particle/momentum/class_4/efficiency": 0.7115384340286255, "metrics/particle/momentum/class_4/event/energy_bias": -0.7120360136032104, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.516512393951416, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1968932151794434, "metrics/particle/momentum/class_4/event/px_bias": 0.3781246244907379, "metrics/particle/momentum/class_4/event/py_bias": -0.22159028053283691, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.341517925262451, "metrics/particle/momentum/class_4/event/pz_bias": -0.09495804458856583, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.262604713439941, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020815521478652954, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15557225048542023, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1538931429386139, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.016854217275977135, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19916626811027527, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0416666679084301, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.908475637435913, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.12775573134422302, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.897852659225464, "metrics/particle/momentum/class_5/accepted/samples": 144.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09957468509674072, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2237776517868042, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.22953884303569794, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05596959963440895, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6481243371963501, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05128205195069313, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.300546169281006, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.10216796398162842, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.990031719207764, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.9056603908538818, "metrics/particle/momentum/class_5/event/energy_bias": -0.48647040128707886, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.443366527557373, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.753869891166687, "metrics/particle/momentum/class_5/event/px_bias": -0.11207178235054016, "metrics/particle/momentum/class_5/event/py_bias": 0.1400376558303833, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3729487657546997, "metrics/particle/momentum/class_5/event/pz_bias": 0.43810003995895386, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.898640155792236, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39569103717803955, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3978753387928009, "metrics/set_slot/angular_recall_all_dr0p10": 0.9404494166374207, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8001784086227417, "metrics/set_slot/soft_count_bias_mean": 0.4371461868286133}, "step_train_time": 5.530242204666138, "step_valid_time": 55966.62883734703, "step_total_time": 55972.159079790115, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.990102672453314, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15629732776241778, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2858, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2833, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9441691443673604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501329787234043, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9471416735708368, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049867021276595744, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9359101420548398, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9418218085106383, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9388566694283347, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9441691443673604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.990175134836126, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1549740795282255, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2114, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2105, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9345711759504863, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9450156459544032, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9397643920871305, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05498435404559678, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9305923961096375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9409924005364327, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9357635030006668, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9345711759504863, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897682769121128, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15977458539584544, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1266, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1168, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1164, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.906832298136646, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9225908372827805, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9146436961628818, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07740916271721959, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9037267080745341, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.919431279620853, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9115113547376664, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.906832298136646} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_19000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..214e5f49ab7dfa3e7044fa4dfd7e979a7e35e584 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.9001326511275346, "train": {"Classification_binary": 0.1835886538028717, "Cardinality": 0.15326476097106934, "Classification": 0.09504614025354385, "Regression_pt": 0.1463555544614792, "Regression_eta": 0.002040407620370388, "Regression_sin_phi": 0.0012398276012390852, "Regression_cos_phi": 0.0011830648873001337, "Regression_energy": 0.14611230790615082, "HitGrouping": 0.17238005995750427, "Auxiliary": 0.23214995861053467, "Total": 1.1911934614181519, "Total_unweighted": 0.5755659565329552}, "train_group": {}, "valid": {"Classification_binary": 0.18755942583084106, "Cardinality": 0.13422332207361856, "Classification": 0.08826955159505208, "Regression_pt": 0.14084986845652261, "Regression_eta": 0.001917642851670583, "Regression_sin_phi": 0.001041195821017027, "Regression_cos_phi": 0.0010573704882214467, "Regression_energy": 0.13980410496393839, "HitGrouping": 0.1667036215464274, "Auxiliary": 0.2235343853632609, "Total": 1.1239763895670574, "Total_unweighted": 0.5604991600072632, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.83528071641922, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5389504432678223, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3271937370300293, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5443943738937378, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5390320420265198, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3648814558982849, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7757662534713745, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.49506568908691406, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3388681709766388, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5017858147621155, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5144421458244324, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46187731623649597, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8520971536636353, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5366812944412231, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3031195104122162, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5392255187034607, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.528130292892456, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33581241965293884, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8398391008377075, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5336787700653076, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.31407293677330017, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5375306606292725, "metrics/hit_grouping/calorimeter/all/object_purity": 0.530410647392273, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3564741611480713, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02630041167140007, "metrics/hit_grouping/calorimeter/background_recall": 0.6427474021911621, "metrics/hit_grouping/calorimeter/completeness": 0.6039960980415344, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6107518076896667, "metrics/hit_grouping/calorimeter/energy_purity": 0.6038684844970703, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36937215924263, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8560982942581177, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5463982820510864, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24550612270832062, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5533762574195862, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5461094975471497, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.308689147233963, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8436289429664612, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5574081540107727, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.331877738237381, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5602304339408875, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5505679249763489, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3577147126197815, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8014745116233826, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.48202741146087646, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4406522512435913, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4803352952003479, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.47617557644844055, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4563266336917877, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8284193277359009, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5678808093070984, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.37626907229423523, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5748009085655212, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5730259418487549, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3942074179649353, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.811696469783783, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6382165551185608, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.48142334818840027, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6345568299293518, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6409041285514832, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40345367789268494, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8557174205780029, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4835323691368103, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2257934957742691, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.485046923160553, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.47045737504959106, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.30989688634872437, "metrics/hit_grouping/calorimeter/purity": 0.5954126119613647, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8231428265571594, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09422516822814941, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0953846126794815, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.095179483294487, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06551683694124222, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8289429545402527, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29159900546073914, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12485747039318085, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2906094193458557, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2816695272922516, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2502850592136383, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8408218026161194, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5569688081741333, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3296169936656952, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5611608624458313, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5546261072158813, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3693926930427551, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7184178829193115, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42124924063682556, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4872213900089264, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5288126468658447, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.606246292591095, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7726842761039734, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3258553147315979, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5257350206375122, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5573920607566833, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47672462463378906, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.666256844997406, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4355359673500061, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4150610566139221, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4516410529613495, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6811057925224304, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6917064785957336, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4217834770679474, "metrics/hit_grouping/tracker/all/object_completeness": 0.4471898078918457, "metrics/hit_grouping/tracker/all/object_purity": 0.4849390983581543, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6402323842048645, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0034771256614476442, "metrics/hit_grouping/tracker/background_recall": 0.024521708488464355, "metrics/hit_grouping/tracker/completeness": 0.4627460837364197, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46233320236206055, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7226487398147583, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40838268399238586, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5218476057052612, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5492093563079834, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6052847504615784, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6689695715904236, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43860650062561035, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4138716757297516, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.462423712015152, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.673710286617279, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6471771001815796, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4336569607257843, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31686004996299744, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3661905825138092, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6822410821914673, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6963928937911987, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43772464990615845, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.45075297355651855, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4932883381843567, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6289539933204651, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6886118650436401, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.42410120368003845, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41813743114471436, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5022298693656921, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6324900388717651, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6861454844474792, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40060874819755554, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4476562738418579, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4710218906402588, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6562865972518921, "metrics/hit_grouping/tracker/purity": 0.44044846296310425, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8841709494590759, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04126846045255661, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11392118781805038, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11845991760492325, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08514335006475449, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6786958575248718, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.271124929189682, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.31847211718559265, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.33182260394096375, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5619491934776306, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6670352220535278, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49064400792121887, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5071319341659546, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5521723031997681, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7245734930038452, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5431063175201416, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6809147000312805, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7698693871498108, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6208680868148804, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.057378944009542465, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09006331861019135, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.172504425048828, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30282655358314514, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1723867505788803, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8481796383857727, "metrics/particle/class_1/pid_accuracy": 0.9653112888336182, "metrics/particle/class_2/efficiency": 0.5005609393119812, "metrics/particle/class_2/pid_accuracy": 0.7149260640144348, "metrics/particle/class_3/efficiency": 0.7180829644203186, "metrics/particle/class_3/pid_accuracy": 0.9637607336044312, "metrics/particle/class_4/efficiency": 0.6843118667602539, "metrics/particle/class_4/pid_accuracy": 0.4542897343635559, "metrics/particle/class_5/efficiency": 0.9039999842643738, "metrics/particle/class_5/pid_accuracy": 0.6061946749687195, "metrics/particle/count/bias_mean": -3.31640625, "metrics/particle/count/mae": 4.665364742279053, "metrics/particle/count/prediction_mean": 66.73958587646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07665344327688217, "metrics/particle/event/energy_response_mean": 0.9509804844856262, "metrics/particle/event/met_abs_error": 6.953612804412842, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06354444473981857, "metrics/particle/event/scalar_pt_response_mean": 0.9663140773773193, "metrics/particle/event/vector_pt_closure": 0.0597180612385273, "metrics/particle/matched/delta_eta_abs_mean": 0.016473472118377686, "metrics/particle/matched/delta_phi_abs_mean": 0.01734357886016369, "metrics/particle/matched/delta_r_mean": 0.026840634644031525, "metrics/particle/matched/energy_relative_abs_mean": 0.1472042202949524, "metrics/particle/matched/pid_accuracy": 0.9446342587471008, "metrics/particle/matched/pt_relative_abs_mean": 0.1463228464126587, "metrics/particle/matching/duplicate_fraction": 0.0635632872581482, "metrics/particle/matching/efficiency": 0.7617047429084778, "metrics/particle/matching/f1": 0.7801711559295654, "metrics/particle/matching/purity": 0.7995551824569702, "metrics/particle/matching/target_coverage_dr0p05": 0.6803895831108093, "metrics/particle/matching/target_coverage_dr0p10": 0.808997631072998, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5740107297897339, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8062654733657837, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0261098500341177, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14635983109474182, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14539237320423126, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00608615018427372, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18378962576389313, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007975460030138493, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7532493472099304, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.032554347068071365, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9806866645812988, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8150.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2854618430137634, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0674611330032349, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2346677780151367, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0709341764450073, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14555390179157257, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.315592765808105, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014903130009770393, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0723652839660645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0016643283888697624, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.896475076675415, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10065.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7452450394630432, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -4.818426609039307, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.752973556518555, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.288607597351074, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.15384384989738464, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8942680954933167, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.32236099243164, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.8815430998802185, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.307272911071777, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0252563264220953, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15457434952259064, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1547386348247528, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014865383505821228, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1923111230134964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030123721808195114, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6145176887512207, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.01027610618621111, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.0572052001953125, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1859.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4542199969291687, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9004892110824585, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1112382411956787, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9224967956542969, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1924651861190796, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.391359329223633, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05978706106543541, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5389392375946045, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1174166202545166, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.596018314361572, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2442.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6153591275215149, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.691866397857666, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.636431694030762, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5557059049606323, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1799834817647934, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09891501814126968, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.404925346374512, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.37408408522605896, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.744827270507812, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027284186333417892, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14549259841442108, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14533771574497223, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009136068634688854, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18531563878059387, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007097232155501842, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2515508532524109, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01325189508497715, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7423451542854309, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16908.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23110440373420715, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3009496927261353, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0337438583374023, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8693633675575256, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14059863984584808, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.148226737976074, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00409697787836194, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4389517903327942, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02104315720498562, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7865006923675537, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20747.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7887297868728638, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.9806180000305176, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.102947235107422, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.743691444396973, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.5124623775482178, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.611546516418457, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.917144775390625, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1483176350593567, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.262853145599365, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026788564398884773, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1463824063539505, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14600352942943573, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.006554967258125544, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1853477507829666, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004941115155816078, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4975886046886444, "metrics/particle/momentum/all/accepted/pz_bias": -0.00082295952597633, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6342802047729492, "metrics/particle/momentum/all/accepted/samples": 26917.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26394122838974, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2008717060089111, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1736830472946167, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0077096223831177, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14590725302696228, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.495606422424316, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01145726814866066, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7848793268203735, "metrics/particle/momentum/all/assigned/pz_bias": 0.021247439086437225, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7794690132141113, "metrics/particle/momentum/all/assigned/samples": 33254.0, "metrics/particle/momentum/all/efficiency": 0.7604961395263672, "metrics/particle/momentum/all/event/energy_bias": -14.490910530090332, "metrics/particle/momentum/all/event/p3_error_mean": 21.048648834228516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.074637413024902, "metrics/particle/momentum/all/event/px_bias": -1.8462897539138794, "metrics/particle/momentum/all/event/py_bias": 2.4068996906280518, "metrics/particle/momentum/all/event/pz_abs_mean": 15.898681640625, "metrics/particle/momentum/all/event/pz_bias": -0.6557766199111938, "metrics/particle/momentum/all/event/pz_residual_rms": 24.0585880279541, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02990609034895897, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13322733342647552, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13409490883350372, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.004282273352146149, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17110905051231384, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005228204187005758, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5360356569290161, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00046524457866325974, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6443017721176147, "metrics/particle/momentum/class_1/accepted/samples": 14154.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.162832111120224, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.36604347825050354, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.38708627223968506, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2233981341123581, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08318375796079636, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.5824508666992188, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008974438533186913, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7016948461532593, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.028684966266155243, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3888957500457764, "metrics/particle/momentum/class_1/assigned/samples": 16157.0, "metrics/particle/momentum/class_1/efficiency": 0.8482052087783813, "metrics/particle/momentum/class_1/event/energy_bias": -0.33461302518844604, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.983373641967773, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.231545448303223, "metrics/particle/momentum/class_1/event/px_bias": -1.3086427450180054, "metrics/particle/momentum/class_1/event/py_bias": 1.9370676279067993, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.681244850158691, "metrics/particle/momentum/class_1/event/pz_bias": 0.022564202547073364, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.77117347717285, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.030608106404542923, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18528814613819122, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18794719874858856, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.018209761008620262, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22789376974105835, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022342586889863014, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4628325700759888, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.006398487836122513, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.605154275894165, "metrics/particle/momentum/class_2/accepted/samples": 1477.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5407620668411255, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5875161290168762, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7998021245002747, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4628450870513916, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25440916419029236, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.506014347076416, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.241633653640747, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06607688963413239, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.7052764892578125, "metrics/particle/momentum/class_2/assigned/samples": 2268.0, "metrics/particle/momentum/class_2/efficiency": 0.4961370527744293, "metrics/particle/momentum/class_2/event/energy_bias": -12.934027671813965, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.851892471313477, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.00821304321289, "metrics/particle/momentum/class_2/event/px_bias": -0.10443907231092453, "metrics/particle/momentum/class_2/event/py_bias": 0.09370233118534088, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.9610595703125, "metrics/particle/momentum/class_2/event/pz_bias": 1.7134591341018677, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.775562286376953, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02219267562031746, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15788331627845764, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1555507332086563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024256091564893723, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.195943221449852, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015602055937051773, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2889100909233093, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008213181979954243, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9854588508605957, "metrics/particle/momentum/class_3/accepted/samples": 10896.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33413586020469666, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.2110719680786133, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.108384609222412, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9666825532913208, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19863252341747284, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.680150985717773, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008721132762730122, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6258276700973511, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.004266378935426474, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4351108074188232, "metrics/particle/momentum/class_3/assigned/samples": 14333.0, "metrics/particle/momentum/class_3/efficiency": 0.716512143611908, "metrics/particle/momentum/class_3/event/energy_bias": 0.9481297135353088, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.30689525604248, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.2397308349609375, "metrics/particle/momentum/class_3/event/px_bias": -0.4247543215751648, "metrics/particle/momentum/class_3/event/py_bias": -0.08915068209171295, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.538686275482178, "metrics/particle/momentum/class_3/event/pz_bias": -2.3043084144592285, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.217092514038086, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02990862913429737, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.156846821308136, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15052448213100433, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03571442514657974, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19744138419628143, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01600000075995922, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8394925594329834, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.38486024737358093, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1186418533325195, "metrics/particle/momentum/class_4/accepted/samples": 250.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.34506678581237793, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.811692953109741, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.070866823196411, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9470232725143433, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21511627733707428, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.371152877807617, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020348837599158287, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1187431812286377, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2622755169868469, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.863150119781494, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.6868131756782532, "metrics/particle/momentum/class_4/event/energy_bias": -0.9051074981689453, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6420788764953613, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8189111948013306, "metrics/particle/momentum/class_4/event/px_bias": 0.15695105493068695, "metrics/particle/momentum/class_4/event/py_bias": 0.060422927141189575, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.578531265258789, "metrics/particle/momentum/class_4/event/pz_bias": 0.11116586625576019, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.882291793823242, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.023430246859788895, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1521194726228714, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15634037554264069, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06625588983297348, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1982567012310028, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.057877779006958, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.32079648971557617, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.410163402557373, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0783073678612709, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.18833422660827637, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.19540581107139587, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.04632636904716492, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06578947603702545, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2736811339855194, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03947368264198303, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.133103132247925, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.38247251510620117, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.4881134033203125, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.8805031180381775, "metrics/particle/momentum/class_5/event/energy_bias": -1.2652931213378906, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.2162282466888428, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7197734117507935, "metrics/particle/momentum/class_5/event/px_bias": -0.16540473699569702, "metrics/particle/momentum/class_5/event/py_bias": 0.4048575758934021, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1068154573440552, "metrics/particle/momentum/class_5/event/pz_bias": -0.19865749776363373, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7745554447174072, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4046371579170227, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.406218945980072, "metrics/set_slot/angular_recall_all_dr0p10": 0.9372246861457825, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7970187664031982, "metrics/set_slot/soft_count_bias_mean": -1.0140212774276733}, "step_train_time": 5.368434906005859, "step_valid_time": 59074.2917265892, "step_total_time": 59079.660161972046, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.983942365693037, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15976890326278415, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3011, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2837, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2823, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9372315824248431, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9422118897376287, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.939715137462736, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0577881102623713, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9326065411298315, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9375622716705414, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9350778403444849, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9372315824248431, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9675999483524882, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1454698660849244, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2238, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2110, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2103, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9328028293545535, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9428060768543343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9377777777777778, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05719392314566577, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9297082228116711, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9396782841823056, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9346666666666666, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9328028293545535, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9663767345623959, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14771945600440672, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1190, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1185, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9239130434782609, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9384858044164038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9311424100156495, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061514195583596214, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9200310559006211, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9345425867507886, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9272300469483568, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9239130434782609} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_2000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..bd67523409d3165fcfaff8b79165de02e649a983 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.09475080538184574, "train": {"Classification_binary": 0.36839020252227783, "Cardinality": 0.21803796291351318, "Classification": 0.2572842836380005, "Regression_pt": 0.41820821166038513, "Regression_eta": 0.005616167560219765, "Regression_sin_phi": 0.0026229703798890114, "Regression_cos_phi": 0.002617057180032134, "Regression_energy": 0.4192269444465637, "HitGrouping": 0.2681645154953003, "Auxiliary": 0.48659032583236694, "Total": 2.4875264167785645, "Total_unweighted": 1.473965837387368}, "train_group": {}, "valid": {"Classification_binary": 0.38184189796447754, "Cardinality": 0.2517293492952983, "Classification": 0.2795287569363912, "Regression_pt": 0.42900721232096356, "Regression_eta": 0.005531663695971171, "Regression_sin_phi": 0.0028169018526872, "Regression_cos_phi": 0.0028055993219216666, "Regression_energy": 0.4253040552139282, "HitGrouping": 0.26481014490127563, "Auxiliary": 0.5094735225041708, "Total": 2.602423667907715, "Total_unweighted": 1.5268360873063405, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7879983186721802, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2052297294139862, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3393784761428833, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.20308443903923035, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.19379740953445435, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5090205073356628, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7513006925582886, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.21840807795524597, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34632328152656555, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.21524710953235626, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.19620868563652039, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5482887029647827, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7928841710090637, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.21946832537651062, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3391178548336029, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.21728815138339996, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.20630410313606262, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4985022246837616, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7875820994377136, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.21479132771492004, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33984801173210144, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.21253442764282227, "metrics/hit_grouping/calorimeter/all/object_purity": 0.20137400925159454, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5063526034355164, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025136396288871765, "metrics/hit_grouping/calorimeter/background_recall": 0.6225994825363159, "metrics/hit_grouping/calorimeter/completeness": 0.31335699558258057, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3427736759185791, "metrics/hit_grouping/calorimeter/energy_purity": 0.33863338828086853, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5611347556114197, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8195844888687134, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.21224062144756317, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26843833923339844, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.21162310242652893, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.19654452800750732, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.43553298711776733, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.770965039730072, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.22498151659965515, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3646288216114044, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.22147336602210999, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.22143155336380005, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5459728240966797, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7371600270271301, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2095712125301361, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.465175062417984, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.20514817535877228, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.1906946450471878, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6149665117263794, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7454349398612976, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.23221543431282043, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4119589030742645, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.23000678420066833, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.23228415846824646, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5978822708129883, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.709952712059021, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.41537314653396606, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6551543474197388, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.40617862343788147, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3905985951423645, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6473050713539124, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8428266644477844, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.170392245054245, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22197669744491577, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.1689712107181549, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.14392989873886108, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3899156153202057, "metrics/hit_grouping/calorimeter/purity": 0.30853497982025146, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8997412323951721, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04592354968190193, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1246153861284256, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04487179219722748, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.008009887300431728, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8648086786270142, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.05036477744579315, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09977194666862488, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.050849609076976776, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.024913135915994644, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.271949827671051, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7807497382164001, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.22784772515296936, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3582437038421631, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2254074662923813, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.2156447023153305, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5297743678092957, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6477419137954712, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38986849784851074, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.23654796183109283, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2254212349653244, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7346084713935852, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7481729984283447, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3707234859466553, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.30596229434013367, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.24233698844909668, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.545709490776062, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5656538605690002, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.38355517387390137, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.19029958546161652, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.19871455430984497, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7956997752189636, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6069709658622742, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.38447004556655884, "metrics/hit_grouping/tracker/all/object_completeness": 0.2146582156419754, "metrics/hit_grouping/tracker/all/object_purity": 0.21082961559295654, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7553372383117676, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011831377632915974, "metrics/hit_grouping/tracker/background_recall": 0.054848071187734604, "metrics/hit_grouping/tracker/completeness": 0.22831812500953674, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.597113311290741, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6007072329521179, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38168564438819885, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.24963584542274475, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2586572468280792, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7680182456970215, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.599808394908905, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.37312012910842896, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.1830359846353531, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.17596979439258575, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7664191722869873, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6284857988357544, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.40271034836769104, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.17061132192611694, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.14169737696647644, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7154126167297363, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6276838779449463, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4138209819793701, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.21965999901294708, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.207905575633049, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7404744625091553, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.648314893245697, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46671104431152344, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.25675591826438904, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.26659348607063293, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7083888053894043, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.572717547416687, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.33177241683006287, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.20073984563350677, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2048334777355194, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7829548120498657, "metrics/hit_grouping/tracker/purity": 0.2191537767648697, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.887464702129364, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.008253692649304867, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.014525067992508411, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.01595398038625717, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07384882867336273, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.659144401550293, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.12908242642879486, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.07921254634857178, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.06184196099638939, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6008294224739075, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5629157423973083, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4646714925765991, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.2572755813598633, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.25427454710006714, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8655332922935486, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4240918457508087, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.3275206387042999, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.3325579762458801, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5461708903312683, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.061635520309209824, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.015366114675998688, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.96698570251465, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.17512579262256622, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.0446859635412693, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.4703007638454437, "metrics/particle/class_1/pid_accuracy": 0.8654129505157471, "metrics/particle/class_2/efficiency": 0.3430558741092682, "metrics/particle/class_2/pid_accuracy": 0.12557226419448853, "metrics/particle/class_3/efficiency": 0.45811253786087036, "metrics/particle/class_3/pid_accuracy": 0.8840029835700989, "metrics/particle/class_4/efficiency": 0.4359961450099945, "metrics/particle/class_4/pid_accuracy": 0.1832229644060135, "metrics/particle/class_5/efficiency": 0.4020000100135803, "metrics/particle/class_5/pid_accuracy": 0.04477611929178238, "metrics/particle/count/bias_mean": -5.67578125, "metrics/particle/count/mae": 8.036458015441895, "metrics/particle/count/prediction_mean": 64.38021087646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.18045674264431, "metrics/particle/event/energy_response_mean": 1.028828740119934, "metrics/particle/event/met_abs_error": 17.95599937438965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.15593212842941284, "metrics/particle/event/scalar_pt_response_mean": 1.0557615756988525, "metrics/particle/event/vector_pt_closure": 0.13487306237220764, "metrics/particle/matched/delta_eta_abs_mean": 0.023213418200612068, "metrics/particle/matched/delta_phi_abs_mean": 0.022344449535012245, "metrics/particle/matched/delta_r_mean": 0.035951972007751465, "metrics/particle/matched/energy_relative_abs_mean": 0.205637589097023, "metrics/particle/matched/pid_accuracy": 0.8175079226493835, "metrics/particle/matched/pt_relative_abs_mean": 0.20043881237506866, "metrics/particle/matching/duplicate_fraction": 0.1135021448135376, "metrics/particle/matching/efficiency": 0.45383158326148987, "metrics/particle/matching/f1": 0.472991943359375, "metrics/particle/matching/purity": 0.4938415288925171, "metrics/particle/matching/target_coverage_dr0p05": 0.3795606195926666, "metrics/particle/matching/target_coverage_dr0p10": 0.5489377975463867, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.28370165824890137, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5384272336959839, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.036000385880470276, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20797285437583923, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20328490436077118, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.017027318477630615, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24381980299949646, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.022522522136569023, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.510008454322815, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.006079796701669693, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.3265881538391113, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4662.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8319859504699707, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.179489850997925, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.085292816162109, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.885626792907715, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.4068138301372528, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 22.2445011138916, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04277223348617554, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.3342602252960205, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02700991928577423, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.300683975219727, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9422.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.42629846930503845, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 8.918671607971191, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 18.953046798706055, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.695096969604492, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.6315155029296875, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.013100147247314453, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 15.94857406616211, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.5532937049865723, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 26.78877067565918, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03381751477718353, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20697136223316193, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.2019629329442978, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0077631413005292416, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24440951645374298, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04413239657878876, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.4666950702667236, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08852913975715637, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.252131462097168, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 997.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.1756316423416138, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.8996169567108154, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8.318653106689453, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8.134733200073242, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.4653916358947754, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 60.775638580322266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08606557548046112, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.108079433441162, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.038792628794908524, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.987628936767578, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2196.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3300231695175171, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -12.741438865661621, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 18.382522583007812, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.6879491806030273, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.080750972032547, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.14779868721961975, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 18.08679962158203, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.7220733165740967, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 38.32219314575195, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0355454720556736, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20627175271511078, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20037223398685455, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.021552134305238724, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24084745347499847, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002401075791567564, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4751771092414856, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.023266378790140152, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2283289432525635, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 10412.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6774905323982239, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.301582336425781, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.507300615310669, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.307443141937256, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.39458081126213074, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.797203063964844, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.01847224310040474, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.2465778589248657, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.060558345168828964, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.4914441108703613, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20409.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4857022762298584, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 16.431251525878906, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 21.68454360961914, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 18.150054931640625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.8820254802703857, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.635448694229126, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 9.730414390563965, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.2611209154129028, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.613801956176758, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03557024151086807, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20680862665176392, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20131583511829376, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0193841103464365, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24193517863750458, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.010826955549418926, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8989172577857971, "metrics/particle/momentum/all/accepted/pz_bias": -0.01134528312832117, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.4287781715393066, "metrics/particle/momentum/all/accepted/samples": 16071.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.7570974826812744, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.8753461837768555, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.007240295410156, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.8085315227508545, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.40303492546081543, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 24.508689880371094, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.030255721881985664, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.7627674341201782, "metrics/particle/momentum/all/assigned/pz_bias": -0.04919634386897087, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.93071985244751, "metrics/particle/momentum/all/assigned/samples": 32027.0, "metrics/particle/momentum/all/efficiency": 0.45406001806259155, "metrics/particle/momentum/all/event/energy_bias": 12.608484268188477, "metrics/particle/momentum/all/event/p3_error_mean": 44.2777099609375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 22.205562591552734, "metrics/particle/momentum/all/event/px_bias": 5.4327898025512695, "metrics/particle/momentum/all/event/py_bias": 0.7701472043991089, "metrics/particle/momentum/all/event/pz_abs_mean": 32.74759292602539, "metrics/particle/momentum/all/event/pz_bias": -7.536487579345703, "metrics/particle/momentum/all/event/pz_residual_rms": 48.33491516113281, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04526640474796295, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.22475476562976837, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21820314228534698, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.020446108654141426, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2569020092487335, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.012849872931838036, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.0899146795272827, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05094829574227333, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.5978410243988037, "metrics/particle/momentum/class_1/accepted/samples": 7860.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.6141259670257568, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.234968662261963, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.327996015548706, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.1064364910125732, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.37450462579727173, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.515768051147461, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.029590489342808723, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.7542189359664917, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07387444376945496, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.066470146179199, "metrics/particle/momentum/class_1/assigned/samples": 15140.0, "metrics/particle/momentum/class_1/efficiency": 0.47102534770965576, "metrics/particle/momentum/class_1/event/energy_bias": 47.59840393066406, "metrics/particle/momentum/class_1/event/p3_error_mean": 34.358028411865234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 19.905866622924805, "metrics/particle/momentum/class_1/event/px_bias": 6.269074440002441, "metrics/particle/momentum/class_1/event/py_bias": 0.09028864651918411, "metrics/particle/momentum/class_1/event/pz_abs_mean": 23.79070472717285, "metrics/particle/momentum/class_1/event/pz_bias": -5.48760986328125, "metrics/particle/momentum/class_1/event/pz_residual_rms": 32.60334396362305, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043681226670742035, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2344386875629425, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.227460578083992, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08751112967729568, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26885107159614563, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029893925413489342, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8161306381225586, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07555412501096725, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.143138885498047, "metrics/particle/momentum/class_2/accepted/samples": 1037.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.865619421005249, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9417231678962708, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.356063961982727, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.9569500684738159, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37136465311050415, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.499983310699463, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.061744965612888336, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.975013017654419, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11196129024028778, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.686326026916504, "metrics/particle/momentum/class_2/assigned/samples": 2235.0, "metrics/particle/momentum/class_2/efficiency": 0.3483372628688812, "metrics/particle/momentum/class_2/event/energy_bias": -34.83594512939453, "metrics/particle/momentum/class_2/event/p3_error_mean": 31.40938377380371, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.342829704284668, "metrics/particle/momentum/class_2/event/px_bias": 0.32163625955581665, "metrics/particle/momentum/class_2/event/py_bias": -0.3583410680294037, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.52098846435547, "metrics/particle/momentum/class_2/event/pz_bias": -1.285139799118042, "metrics/particle/momentum/class_2/event/pz_residual_rms": 36.3238525390625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023174060508608818, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1817954182624817, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1776202768087387, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.006315123289823532, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2184726893901825, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.004033419769257307, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.5012815594673157, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01435726135969162, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.6367812156677246, "metrics/particle/momentum/class_3/accepted/samples": 6942.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8928546905517578, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 6.064462661743164, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 6.2099223136901855, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 6.070698261260986, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4381959140300751, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 35.01975631713867, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.024242423474788666, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5346561670303345, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.044403355568647385, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.339789867401123, "metrics/particle/momentum/class_3/assigned/samples": 14190.0, "metrics/particle/momentum/class_3/efficiency": 0.4565002918243408, "metrics/particle/momentum/class_3/event/energy_bias": 9.53499984741211, "metrics/particle/momentum/class_3/event/p3_error_mean": 24.424625396728516, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 13.62016487121582, "metrics/particle/momentum/class_3/event/px_bias": 0.4559612274169922, "metrics/particle/momentum/class_3/event/py_bias": -0.2988896667957306, "metrics/particle/momentum/class_3/event/pz_abs_mean": 17.21671485900879, "metrics/particle/momentum/class_3/event/pz_bias": -0.8075079917907715, "metrics/particle/momentum/class_3/event/pz_residual_rms": 32.30351257324219, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04138264060020447, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2240147441625595, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.218252032995224, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.06382250040769577, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.25807318091392517, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.07975459843873978, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.6656365394592285, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06704894453287125, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 5.5213704109191895, "metrics/particle/momentum/class_4/accepted/samples": 163.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.8216559886932373, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.817102909088135, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.355367660522461, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.146211624145508, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4256559908390045, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.330671310424805, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0641399398446083, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.6822237968444824, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09328009933233261, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.608012676239014, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.44780218601226807, "metrics/particle/momentum/class_4/event/energy_bias": -3.931540012359619, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.860163688659668, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.266733646392822, "metrics/particle/momentum/class_4/event/px_bias": -0.746067464351654, "metrics/particle/momentum/class_4/event/py_bias": 0.8555830717086792, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.2280795574188232, "metrics/particle/momentum/class_4/event/pz_bias": 0.3338189125061035, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.972530364990234, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04258392006158829, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2231569141149521, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22867445647716522, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08440470695495605, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.26301753520965576, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.014492753893136978, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.1890690326690674, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4228534698486328, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.374398708343506, "metrics/particle/momentum/class_5/accepted/samples": 69.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.5344516634941101, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.038175344467163, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9037079215049744, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.31316518783569336, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.36974790692329407, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.282106876373291, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1428571492433548, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.63316535949707, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.380731463432312, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 10.44023323059082, "metrics/particle/momentum/class_5/assigned/samples": 119.0, "metrics/particle/momentum/class_5/efficiency": 0.43396225571632385, "metrics/particle/momentum/class_5/event/energy_bias": -5.7574381828308105, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.403937339782715, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.334383964538574, "metrics/particle/momentum/class_5/event/px_bias": -0.8678146004676819, "metrics/particle/momentum/class_5/event/py_bias": 0.48150622844696045, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.5695087909698486, "metrics/particle/momentum/class_5/event/pz_bias": -0.29004910588264465, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.231343269348145, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7112997770309448, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7112263441085815, "metrics/set_slot/angular_recall_all_dr0p10": 0.8104473948478699, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5808133482933044, "metrics/set_slot/soft_count_bias_mean": 1.2718857526779175}, "step_train_time": 5.855871915817261, "step_valid_time": 5873.3350286483765, "step_total_time": 5879.1909012794495, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1502538864919554, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.33424414315891515, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2934, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2429, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2159, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8024446646845061, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8278800272665303, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8149639322261365, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17211997273346966, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7132474397092832, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7358554873892297, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7243751048481798, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8024446646845061, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1169877902476018, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3117743093436961, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2221, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1825, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1705, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8068081343943413, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8217019360648357, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8141869283961632, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17829806393516434, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7537577365163572, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7676722197208464, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.760651349542717, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8068081343943413, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0692340381923546, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3228442025189837, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 992, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 894, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7701863354037267, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8259783513738551, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7971072719967859, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1740216486261449, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6940993788819876, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7443796835970025, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7183607874648453, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7701863354037267} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_20000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..c315bbd17c79a61d39cca184b90b256024b95a0f --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9475080538184575, "train": {"Classification_binary": 0.15905635058879852, "Cardinality": 0.128745898604393, "Classification": 0.08448303490877151, "Regression_pt": 0.1421295553445816, "Regression_eta": 0.0016377945430576801, "Regression_sin_phi": 0.0009468131465837359, "Regression_cos_phi": 0.0009874064708128572, "Regression_energy": 0.13972608745098114, "HitGrouping": 0.16493698954582214, "Auxiliary": 0.2104201465845108, "Total": 1.053351879119873, "Total_unweighted": 0.528967042453587}, "train_group": {}, "valid": {"Classification_binary": 0.17802788813908896, "Cardinality": 0.12184194723765056, "Classification": 0.08616526921590169, "Regression_pt": 0.13844144344329834, "Regression_eta": 0.0017865011468529701, "Regression_sin_phi": 0.0010070027007410924, "Regression_cos_phi": 0.0009886020173629124, "Regression_energy": 0.13759849468866983, "HitGrouping": 0.1641800800959269, "Auxiliary": 0.21385866403579712, "Total": 1.0747357209523518, "Total_unweighted": 0.5440152013519158, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8379597067832947, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5520006418228149, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3152923285961151, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5562030076980591, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5518702268600464, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.36818739771842957, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.788923978805542, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.522031307220459, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3348017632961273, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5292191505432129, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5257164835929871, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.45035579800605774, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8531917333602905, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5408285856246948, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3123127818107605, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5438782572746277, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5370247960090637, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.34350791573524475, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8425256609916687, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5427376627922058, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3152887523174286, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5465294122695923, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5407876372337341, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.36103343963623047, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02580246701836586, "metrics/hit_grouping/calorimeter/background_recall": 0.6367202997207642, "metrics/hit_grouping/calorimeter/completeness": 0.6083469390869141, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6170836091041565, "metrics/hit_grouping/calorimeter/energy_purity": 0.6098700165748596, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36098673939704895, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8638733625411987, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5612966418266296, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25243356823921204, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5688479542732239, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.55682772397995, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3070170283317566, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8434035181999207, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.563922643661499, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3258126974105835, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5664847493171692, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5644721984863281, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.36693352460861206, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7964242696762085, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.48137763142585754, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.43736568093299866, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.47849297523498535, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4821545481681824, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4692200720310211, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8238147497177124, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5657566785812378, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3709747791290283, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5721548795700073, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5819225311279297, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4105885922908783, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8153266310691833, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6472238302230835, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4871794879436493, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.642895519733429, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6418562531471252, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40816327929496765, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8661214709281921, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5046203136444092, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23342707753181458, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5066491961479187, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48363199830055237, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3017275929450989, "metrics/hit_grouping/calorimeter/purity": 0.5993950963020325, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8569383025169373, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.13392403721809387, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11384615302085876, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1330818235874176, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.10153544694185257, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8555887341499329, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30936214327812195, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12428734451532364, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.30893808603286743, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2941262722015381, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24800455570220947, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8414669632911682, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.564873993396759, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33056795597076416, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5690069794654846, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.564336895942688, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.37437695264816284, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.733646035194397, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42035266757011414, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5058750510215759, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5287541747093201, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5842797160148621, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.803506076335907, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3651149868965149, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5564997792243958, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5318155288696289, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41110488772392273, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6689526438713074, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4378199577331543, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42839106917381287, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4675489366054535, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6766165494918823, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7007322311401367, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4261760711669922, "metrics/hit_grouping/tracker/all/object_completeness": 0.4636707305908203, "metrics/hit_grouping/tracker/all/object_purity": 0.49230676889419556, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6250708699226379, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004758172202855349, "metrics/hit_grouping/tracker/background_recall": 0.03861872851848602, "metrics/hit_grouping/tracker/completeness": 0.48057714104652405, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4627102017402649, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7310886383056641, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4164920151233673, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5440996885299683, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5586242079734802, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.587699294090271, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6765649914741516, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44051018357276917, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42786112427711487, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4756137430667877, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6672377586364746, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.659022867679596, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43244338035583496, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32317718863487244, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3628275990486145, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6632281541824341, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7049389481544495, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4317038059234619, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.461212694644928, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5052987933158875, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6147555708885193, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7084330320358276, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45073235034942627, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4479905664920807, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4975798726081848, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6038615107536316, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6938977837562561, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.41465699672698975, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.46963000297546387, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.47445428371429443, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6422383785247803, "metrics/hit_grouping/tracker/purity": 0.45800912380218506, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.888042688369751, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04257167503237724, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1289117932319641, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1332755833864212, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08166811615228653, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6861142516136169, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2628304958343506, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3184591829776764, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3256014883518219, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5469154715538025, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6769443154335022, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4969010055065155, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5256935358047485, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5600751638412476, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7078094482421875, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5577670335769653, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6720762252807617, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7595738172531128, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6467463374137878, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0720396563410759, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.11594157665967941, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.175758361816406, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28232380747795105, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8437499403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15188394486904144, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8572022318840027, "metrics/particle/class_1/pid_accuracy": 0.9635990262031555, "metrics/particle/class_2/efficiency": 0.4974197745323181, "metrics/particle/class_2/pid_accuracy": 0.7304916381835938, "metrics/particle/class_3/efficiency": 0.7442129850387573, "metrics/particle/class_3/pid_accuracy": 0.9647737145423889, "metrics/particle/class_4/efficiency": 0.6852743029594421, "metrics/particle/class_4/pid_accuracy": 0.5, "metrics/particle/class_5/efficiency": 0.9200000166893005, "metrics/particle/class_5/pid_accuracy": 0.6326087117195129, "metrics/particle/count/bias_mean": -2.1803386211395264, "metrics/particle/count/mae": 4.060546875, "metrics/particle/count/prediction_mean": 67.87564849853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07363579422235489, "metrics/particle/event/energy_response_mean": 0.9638276696205139, "metrics/particle/event/met_abs_error": 6.828446865081787, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05763661488890648, "metrics/particle/event/scalar_pt_response_mean": 0.9815695285797119, "metrics/particle/event/vector_pt_closure": 0.05712965503334999, "metrics/particle/matched/delta_eta_abs_mean": 0.01567460037767887, "metrics/particle/matched/delta_phi_abs_mean": 0.01598319225013256, "metrics/particle/matched/delta_r_mean": 0.02512715943157673, "metrics/particle/matched/energy_relative_abs_mean": 0.14318181574344635, "metrics/particle/matched/pid_accuracy": 0.9459572434425354, "metrics/particle/matched/pt_relative_abs_mean": 0.14372673630714417, "metrics/particle/matching/duplicate_fraction": 0.06021657958626747, "metrics/particle/matching/efficiency": 0.7770849466323853, "metrics/particle/matching/f1": 0.7893686294555664, "metrics/particle/matching/purity": 0.8020468354225159, "metrics/particle/matching/target_coverage_dr0p05": 0.7084549069404602, "metrics/particle/matching/target_coverage_dr0p10": 0.8252421021461487, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5904317498207092, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8212088346481323, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023994650691747665, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1419840157032013, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14416058361530304, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.025086840614676476, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18501292169094086, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007735073566436768, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7066471576690674, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0496663898229599, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8940420150756836, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8274.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28505805134773254, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9807795286178589, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0898886919021606, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9302956461906433, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14278747141361237, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.785574913024902, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01560518890619278, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0589385032653809, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01872071623802185, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2563579082489014, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10253.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7565837502479553, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.402789354324341, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.966775894165039, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9712884426116943, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.09586862474679947, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5496495366096497, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.6984663009643555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1145368814468384, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.869584083557129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02286858856678009, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14284710586071014, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15000152587890625, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0345577709376812, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1907484233379364, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02706923522055149, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4949331283569336, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05275016650557518, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.329956531524658, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1921.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4576188027858734, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9205499291419983, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.783480167388916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.622626781463623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1876484602689743, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 11.670835494995117, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06254948675632477, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4648728370666504, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.04152730479836464, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.444353103637695, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2526.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6358821392059326, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.781230926513672, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.57138729095459, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.387770414352417, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.060600947588682175, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1552627682685852, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.417219161987305, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.19741246104240417, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.085174560546875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02573416568338871, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1431693434715271, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14245334267616272, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01338847167789936, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18285882472991943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007545855478383601, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23820050060749054, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005876578390598297, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7363185286521912, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17228.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20929332077503204, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9424030780792236, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7495566010475159, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5828165411949158, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13153110444545746, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.5181803703308105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003732057521119714, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4035957157611847, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0027637288440018892, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6047348976135254, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20900.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8036572337150574, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.0813379287719727, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.763323783874512, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.543972015380859, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5514779090881348, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.4881693124771118, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6604950428009033, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.09359955042600632, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.840427398681641, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025008589029312134, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14278912544250488, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14349719882011414, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018400998786091805, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18407315015792847, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004704080522060394, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4675741195678711, "metrics/particle/momentum/all/accepted/pz_bias": 0.022372256964445114, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6541862487792969, "metrics/particle/momentum/all/accepted/samples": 27423.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2509835660457611, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9524470567703247, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9307114481925964, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7665886878967285, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13916684687137604, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.292341232299805, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011758068576455116, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.757703959941864, "metrics/particle/momentum/all/assigned/pz_bias": 0.010528916493058205, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8179867267608643, "metrics/particle/momentum/all/assigned/samples": 33679.0, "metrics/particle/momentum/all/efficiency": 0.7747923135757446, "metrics/particle/momentum/all/event/energy_bias": -10.265357971191406, "metrics/particle/momentum/all/event/p3_error_mean": 19.995519638061523, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.808059692382812, "metrics/particle/momentum/all/event/px_bias": -0.7079474329948425, "metrics/particle/momentum/all/event/py_bias": 1.8825560808181763, "metrics/particle/momentum/all/event/pz_abs_mean": 14.86756420135498, "metrics/particle/momentum/all/event/pz_bias": -1.2183499336242676, "metrics/particle/momentum/all/event/pz_residual_rms": 23.323076248168945, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026225965470075607, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1257910430431366, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12900303304195404, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.016881586983799934, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16840602457523346, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004410529043525457, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4855887293815613, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.024192379787564278, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5493557453155518, "metrics/particle/momentum/class_1/accepted/samples": 14284.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14507485926151276, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.29574596881866455, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3403645157814026, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19408383965492249, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08265632390975952, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6967549324035645, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007508616428822279, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6340880990028381, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.028795825317502022, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.119270086288452, "metrics/particle/momentum/class_1/assigned/samples": 16248.0, "metrics/particle/momentum/class_1/efficiency": 0.8559957146644592, "metrics/particle/momentum/class_1/event/energy_bias": 0.979263961315155, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.636882781982422, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.343287467956543, "metrics/particle/momentum/class_1/event/px_bias": -0.3597950041294098, "metrics/particle/momentum/class_1/event/py_bias": 0.39700812101364136, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.682866096496582, "metrics/particle/momentum/class_1/event/pz_bias": 0.17798198759555817, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.52120590209961, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029779689386487007, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1873134821653366, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18720082938671112, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0036580353043973446, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2295767068862915, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025273224338889122, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4608763456344604, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10750244557857513, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7028110027313232, "metrics/particle/momentum/class_2/accepted/samples": 1464.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6043927669525146, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6475663185119629, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8394521474838257, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.49505266547203064, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2767857015132904, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.674320697784424, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0535714291036129, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4137587547302246, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05719124525785446, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.202874183654785, "metrics/particle/momentum/class_2/assigned/samples": 2352.0, "metrics/particle/momentum/class_2/efficiency": 0.4917702376842499, "metrics/particle/momentum/class_2/event/energy_bias": -10.848172187805176, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.404998779296875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.714271545410156, "metrics/particle/momentum/class_2/event/px_bias": -0.2693873941898346, "metrics/particle/momentum/class_2/event/py_bias": 0.07278554141521454, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.477468490600586, "metrics/particle/momentum/class_2/event/pz_bias": -1.5817911624908447, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.97216796875, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022883443161845207, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15803921222686768, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15578927099704742, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023041857406497, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1956123560667038, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018638501642271876, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2906256318092346, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02915179170668125, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2493295669555664, "metrics/particle/momentum/class_3/accepted/samples": 11267.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3119182586669922, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7032294273376465, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.567151665687561, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4120073318481445, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1789170205593109, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.464735984802246, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009127719327807426, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6060543656349182, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.01510227657854557, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5886924266815186, "metrics/particle/momentum/class_3/assigned/samples": 14571.0, "metrics/particle/momentum/class_3/efficiency": 0.7409088015556335, "metrics/particle/momentum/class_3/event/energy_bias": 0.9897547960281372, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.82829475402832, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.078080177307129, "metrics/particle/momentum/class_3/event/px_bias": -0.017489977180957794, "metrics/particle/momentum/class_3/event/py_bias": 1.053022861480713, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.224792957305908, "metrics/particle/momentum/class_3/event/pz_bias": -0.1777045726776123, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.758923530578613, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02720898948609829, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16424240171909332, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15889818966388702, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05085211247205734, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20244517922401428, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011450381949543953, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8250336647033691, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3465209901332855, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1224253177642822, "metrics/particle/momentum/class_4/accepted/samples": 262.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.33954256772994995, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.580308675765991, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.7934067249298096, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.666938066482544, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2148997187614441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.65643310546875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022922636941075325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1507222652435303, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.01550309918820858, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2469489574432373, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.7197802066802979, "metrics/particle/momentum/class_4/event/energy_bias": -0.8337066173553467, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6800456047058105, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7962977886199951, "metrics/particle/momentum/class_4/event/px_bias": 0.1127517893910408, "metrics/particle/momentum/class_4/event/py_bias": 0.16880956292152405, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6541017293930054, "metrics/particle/momentum/class_4/event/pz_bias": 0.20739474892616272, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.452206134796143, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01811514049768448, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1439812034368515, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1470787674188614, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.02812649868428707, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18593327701091766, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.034246575087308884, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7587271928787231, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04172844439744949, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.617863416671753, "metrics/particle/momentum/class_5/accepted/samples": 146.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06733502447605133, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1938541978597641, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.19454362988471985, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.03155617415904999, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.069182388484478, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.29702043533325195, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.044025156646966934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9274756908416748, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.14037169516086578, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.084542751312256, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9182389974594116, "metrics/particle/momentum/class_5/event/energy_bias": -0.5524981617927551, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0817766189575195, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5768585205078125, "metrics/particle/momentum/class_5/event/px_bias": -0.17402687668800354, "metrics/particle/momentum/class_5/event/py_bias": 0.19092999398708344, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.057236671447754, "metrics/particle/momentum/class_5/event/pz_bias": 0.15576916933059692, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.9368813037872314, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3861604928970337, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3881227374076843, "metrics/set_slot/angular_recall_all_dr0p10": 0.9422429800033569, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8115439414978027, "metrics/set_slot/soft_count_bias_mean": -0.18860574066638947}, "step_train_time": 5.4036736488342285, "step_valid_time": 62060.00290250778, "step_total_time": 62065.40657663345, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9877046846322637, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1470439808139593, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2862, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2842, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9454905847373637, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458030403172505, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456467867173303, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054196959682749506, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9388833828873472, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9391936549900859, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9390384933091029, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9454905847373637, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.986318179305915, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13586085880522125, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2130, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2120, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9416445623342176, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9479305740987984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9447771124417831, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0520694259012016, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9372236958443855, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9434801958166444, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9403415391439344, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9416445623342176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933575794815459, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14685775997543649, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1192, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9254658385093167, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9349019607843138, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9301599687865783, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06509803921568627, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9208074534161491, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9301960784313725, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.925477955520874, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9254658385093167} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_21000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..ada66b2e6b67d6e349c592e7b43e61247b52204e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9948834565093804, "train": {"Classification_binary": 0.1530640423297882, "Cardinality": 0.13262289762496948, "Classification": 0.07919459789991379, "Regression_pt": 0.13165679574012756, "Regression_eta": 0.0016407864168286324, "Regression_sin_phi": 0.0009460196015425026, "Regression_cos_phi": 0.0009444180759601295, "Regression_energy": 0.1308150589466095, "HitGrouping": 0.15949741005897522, "Auxiliary": 0.20633913576602936, "Total": 1.0250762701034546, "Total_unweighted": 0.4982617190107703}, "train_group": {}, "valid": {"Classification_binary": 0.17652549346288046, "Cardinality": 0.12560619910558066, "Classification": 0.08517298102378845, "Regression_pt": 0.13469995061556497, "Regression_eta": 0.0017711161635816097, "Regression_sin_phi": 0.0009984055844446023, "Regression_cos_phi": 0.000988039964189132, "Regression_energy": 0.13358956575393677, "HitGrouping": 0.1630597710609436, "Auxiliary": 0.21503490209579468, "Total": 1.0712459087371826, "Total_unweighted": 0.533745552568386, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.842353343963623, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5528039336204529, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33078303933143616, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5585508346557617, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5471838712692261, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34948521852493286, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7896202802658081, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5118800401687622, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33107420802116394, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5186989903450012, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5174521803855896, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4320569336414337, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8531980514526367, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5352196097373962, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3036876320838928, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5377752184867859, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5271798968315125, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3318355679512024, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8440057039260864, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5387846827507019, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3148632347583771, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5427496433258057, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5327444672584534, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3465045690536499, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026438331231474876, "metrics/hit_grouping/calorimeter/background_recall": 0.6440244317054749, "metrics/hit_grouping/calorimeter/completeness": 0.6121488213539124, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6236529350280762, "metrics/hit_grouping/calorimeter/energy_purity": 0.6165134906768799, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3674319088459015, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8626198768615723, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5572546124458313, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2564944624900818, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5641971826553345, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5465105772018433, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2915497124195099, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8477917313575745, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5561637878417969, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3256914019584656, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5596820116043091, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5532212257385254, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3543182909488678, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8006598353385925, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4815792441368103, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.42712679505348206, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.47970712184906006, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.48226749897003174, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4546833634376526, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8299993276596069, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.567456841468811, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3710370659828186, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5737608075141907, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5771323442459106, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3923388421535492, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8205543756484985, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6602187156677246, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.49764519929885864, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.656952977180481, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6476322412490845, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3898482322692871, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8620642423629761, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4924212098121643, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2310834378004074, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4947967529296875, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4703233242034912, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2916834056377411, "metrics/hit_grouping/calorimeter/purity": 0.6035349369049072, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8183730840682983, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.12059614807367325, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12923076748847961, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.12116484344005585, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08812279254198074, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.026153845712542534, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8287377953529358, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2989894449710846, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1294184774160385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2977425456047058, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28151607513427734, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23318129777908325, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8454301357269287, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.561489999294281, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32948583364486694, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5658271312713623, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5566708445549011, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3598504662513733, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7314280867576599, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.409444123506546, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5033925771713257, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5410241484642029, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5833831429481506, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7991485595703125, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3079080283641815, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.545628011226654, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5597265362739563, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42512619495391846, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6682356595993042, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43577224016189575, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4231874346733093, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45808306336402893, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6788217425346375, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6992341876029968, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4166824221611023, "metrics/hit_grouping/tracker/all/object_completeness": 0.45884987711906433, "metrics/hit_grouping/tracker/all/object_purity": 0.49285879731178284, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6272907853126526, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0041023981757462025, "metrics/hit_grouping/tracker/background_recall": 0.0331694595515728, "metrics/hit_grouping/tracker/completeness": 0.4749464690685272, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4588460624217987, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7344900369644165, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4053758680820465, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5372675657272339, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5541089773178101, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.585603654384613, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6715284585952759, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43023034930229187, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42203855514526367, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4776095151901245, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6678088903427124, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6504086852073669, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42738673090934753, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32388535141944885, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3730944097042084, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6767799258232117, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6999158263206482, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4260424077510834, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.456513911485672, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.504052996635437, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6222142577171326, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6825380325317383, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4221038520336151, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.43144872784614563, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5222913026809692, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6424766778945923, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7012820839881897, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40353548526763916, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4667111039161682, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.47310033440589905, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6312339305877686, "metrics/hit_grouping/tracker/purity": 0.45234477519989014, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8846304416656494, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03953084349632263, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11111875623464584, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1133895218372345, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07515203952789307, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.686698853969574, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26645931601524353, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32402780652046204, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3333493173122406, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5598756074905396, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6754693984985352, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48503631353378296, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5214523077011108, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5625850558280945, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7099934816360474, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5253493189811707, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6603628396987915, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7436268925666809, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6375152468681335, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.039621997624635696, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.10671049356460571, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.821086883544922, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.31171515583992004, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1812753528356552, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8646418452262878, "metrics/particle/class_1/pid_accuracy": 0.9665209650993347, "metrics/particle/class_2/efficiency": 0.5011218190193176, "metrics/particle/class_2/pid_accuracy": 0.734721302986145, "metrics/particle/class_3/efficiency": 0.7334648966789246, "metrics/particle/class_3/pid_accuracy": 0.9648424983024597, "metrics/particle/class_4/efficiency": 0.706448495388031, "metrics/particle/class_4/pid_accuracy": 0.4850136339664459, "metrics/particle/class_5/efficiency": 0.9039999842643738, "metrics/particle/class_5/pid_accuracy": 0.6327433586120605, "metrics/particle/count/bias_mean": -2.6927082538604736, "metrics/particle/count/mae": 4.322916507720947, "metrics/particle/count/prediction_mean": 67.36328125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06555629521608353, "metrics/particle/event/energy_response_mean": 0.9866594672203064, "metrics/particle/event/met_abs_error": 6.46555757522583, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053609758615493774, "metrics/particle/event/scalar_pt_response_mean": 0.9988728165626526, "metrics/particle/event/vector_pt_closure": 0.05341379716992378, "metrics/particle/matched/delta_eta_abs_mean": 0.015954380854964256, "metrics/particle/matched/delta_phi_abs_mean": 0.016510020941495895, "metrics/particle/matched/delta_r_mean": 0.02574487216770649, "metrics/particle/matched/energy_relative_abs_mean": 0.14179706573486328, "metrics/particle/matched/pid_accuracy": 0.947403073310852, "metrics/particle/matched/pt_relative_abs_mean": 0.1429390013217926, "metrics/particle/matching/duplicate_fraction": 0.05965014174580574, "metrics/particle/matching/efficiency": 0.776360034942627, "metrics/particle/matching/f1": 0.7915726900100708, "metrics/particle/matching/purity": 0.8073934316635132, "metrics/particle/matching/target_coverage_dr0p05": 0.7029998302459717, "metrics/particle/matching/target_coverage_dr0p10": 0.8235599994659424, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5916491746902466, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8197777271270752, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02458101138472557, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13807402551174164, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14043113589286804, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.030179526656866074, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18048101663589478, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006645306944847107, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6910063028335571, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06561733782291412, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7823935747146606, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8427.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2527490556240082, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9232937693595886, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0063360929489136, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8640915155410767, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13339853286743164, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.940876960754395, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014180929400026798, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0201562643051147, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.030449161306023598, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0716354846954346, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10225.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7705742716789246, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.007848277688026428, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.601211547851562, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.7234585285186768, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06543733179569244, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.11002017557621002, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.464166641235352, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6109440326690674, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.329631805419922, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02457498572766781, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14127801358699799, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14871706068515778, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.05074218660593033, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19129374623298645, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023784901946783066, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4011058807373047, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0014114143559709191, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.599071741104126, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1934.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.39600032567977905, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9897904396057129, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.0753021240234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9451894760131836, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1809561401605606, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.133798599243164, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05175819993019104, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2245888710021973, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04338952153921127, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.476792335510254, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2531.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6401853561401367, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.9790658950805664, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.336212158203125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1295257806777954, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03096911311149597, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09529175609350204, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.20485782623291, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5478185415267944, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.744335174560547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0261729434132576, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14285720884799957, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1432296335697174, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015613993629813194, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18327899277210236, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007034822483547032, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24934524297714233, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.043628983199596405, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7794575095176697, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17058.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23171328008174896, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2379250526428223, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9282360672950745, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7634339332580566, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13373129069805145, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.349072456359863, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004139989614486694, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.45788365602493286, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05648919194936752, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.043224811553955, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20773.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.795727014541626, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.35602495074272156, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.882161140441895, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.380672454833984, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.34651410579681396, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.17328806221485138, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.109384059906006, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.4844183921813965, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.7275896072387695, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025570964440703392, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14127574861049652, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1427565962076187, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022568354383111, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18300308287143707, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004157701041549444, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4663254916667938, "metrics/particle/momentum/all/accepted/pz_bias": -0.04720998927950859, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5060006380081177, "metrics/particle/momentum/all/accepted/samples": 27419.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25052985548973083, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1232441663742065, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0386419296264648, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8833374977111816, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1371946632862091, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.92049503326416, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01079662423580885, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7627173066139221, "metrics/particle/momentum/all/assigned/pz_bias": -0.04755917191505432, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7799317836761475, "metrics/particle/momentum/all/assigned/samples": 33529.0, "metrics/particle/momentum/all/efficiency": 0.7746793031692505, "metrics/particle/momentum/all/event/energy_bias": -3.6308891773223877, "metrics/particle/momentum/all/event/p3_error_mean": 19.417577743530273, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.145608901977539, "metrics/particle/momentum/all/event/px_bias": -0.3120459020137787, "metrics/particle/momentum/all/event/py_bias": 0.18801651895046234, "metrics/particle/momentum/all/event/pz_abs_mean": 14.724874496459961, "metrics/particle/momentum/all/event/pz_bias": -4.643180847167969, "metrics/particle/momentum/all/event/pz_residual_rms": 22.36661720275879, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028567606583237648, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1250641942024231, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12805518507957458, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.024899594485759735, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1663007289171219, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003453277051448822, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4793122410774231, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0721881091594696, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4163564443588257, "metrics/particle/momentum/class_1/accepted/samples": 14479.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1433330774307251, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3486085534095764, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3745039999485016, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2382947951555252, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07192075997591019, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4642250537872314, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007567367982119322, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6553383469581604, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06835025548934937, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.24468731880188, "metrics/particle/momentum/class_1/assigned/samples": 16254.0, "metrics/particle/momentum/class_1/efficiency": 0.8676814436912537, "metrics/particle/momentum/class_1/event/energy_bias": 9.159687042236328, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.670034408569336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.750259399414062, "metrics/particle/momentum/class_1/event/px_bias": -0.34443050622940063, "metrics/particle/momentum/class_1/event/py_bias": 0.32650652527809143, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.539206504821777, "metrics/particle/momentum/class_1/event/pz_bias": -2.795771598815918, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.933170318603516, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029735330492258072, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18643900752067566, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18949557840824127, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022410769015550613, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2303617000579834, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024187974631786346, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5162955522537231, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09324625879526138, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5868422985076904, "metrics/particle/momentum/class_2/accepted/samples": 1447.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5938491225242615, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6919071674346924, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7990232706069946, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4545306861400604, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26828211545944214, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.274072647094727, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04932929575443268, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.387094736099243, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13248445093631744, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.863666534423828, "metrics/particle/momentum/class_2/assigned/samples": 2311.0, "metrics/particle/momentum/class_2/efficiency": 0.4860597848892212, "metrics/particle/momentum/class_2/event/energy_bias": -13.712791442871094, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.830424308776855, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.353791236877441, "metrics/particle/momentum/class_2/event/px_bias": 1.0728356838226318, "metrics/particle/momentum/class_2/event/py_bias": -1.1312026977539062, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.24035358428955, "metrics/particle/momentum/class_2/event/pz_bias": -1.0308184623718262, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.90378189086914, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02112681046128273, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15578754246234894, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15517733991146088, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024212464690208435, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19552063941955566, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019846640061587095, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28870320320129395, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009935145266354084, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0380905866622925, "metrics/particle/momentum/class_3/accepted/samples": 11085.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31595689058303833, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.075951099395752, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8357547521591187, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.688076138496399, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18883585929870605, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.12118148803711, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007885453291237354, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6052772402763367, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.046793133020401, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.528231620788574, "metrics/particle/momentum/class_3/assigned/samples": 14457.0, "metrics/particle/momentum/class_3/efficiency": 0.7289406061172485, "metrics/particle/momentum/class_3/event/energy_bias": 1.2202590703964233, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.739858627319336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.132174491882324, "metrics/particle/momentum/class_3/event/px_bias": -0.7000876069068909, "metrics/particle/momentum/class_3/event/py_bias": 0.9830425381660461, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.1243391036987305, "metrics/particle/momentum/class_3/event/pz_bias": -1.077406883239746, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.959009170532227, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027379516512155533, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16243848204612732, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15734241902828217, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07528179883956909, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20515361428260803, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0037878789007663727, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7208167314529419, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.18920591473579407, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.796943187713623, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3437162935733795, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.00758695602417, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.9094386696815491, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8093345165252686, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1994301974773407, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.383302688598633, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005698005668818951, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9038975238800049, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04383331537246704, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.751917839050293, "metrics/particle/momentum/class_4/assigned/samples": 351.0, "metrics/particle/momentum/class_4/efficiency": 0.7252747416496277, "metrics/particle/momentum/class_4/event/energy_bias": -0.44909176230430603, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.7164406776428223, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8190419673919678, "metrics/particle/momentum/class_4/event/px_bias": 0.02197999879717827, "metrics/particle/momentum/class_4/event/py_bias": -0.11419597268104553, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6660164594650269, "metrics/particle/momentum/class_4/event/pz_bias": 0.19541049003601074, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.7852783203125, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02120806835591793, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16159242391586304, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1684228926897049, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01693985052406788, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20710043609142303, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0416666679084301, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8164422512054443, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3759109079837799, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.463315725326538, "metrics/particle/momentum/class_5/accepted/samples": 144.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.060650017112493515, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.19413648545742035, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2062593549489975, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03311614319682121, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07051282376050949, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2789464592933655, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1598997116088867, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6278465986251831, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.15271520614624, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.9056603908538818, "metrics/particle/momentum/class_5/event/energy_bias": 0.15104660391807556, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.480915069580078, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8211075067520142, "metrics/particle/momentum/class_5/event/px_bias": -0.3623434901237488, "metrics/particle/momentum/class_5/event/py_bias": 0.12386618554592133, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3254752159118652, "metrics/particle/momentum/class_5/event/pz_bias": 0.06540550291538239, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.307020664215088, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3892152011394501, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.39195388555526733, "metrics/set_slot/angular_recall_all_dr0p10": 0.9430979490280151, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8093693852424622, "metrics/set_slot/soft_count_bias_mean": -0.6711347103118896}, "step_train_time": 5.253698348999023, "step_valid_time": 65072.38629412651, "step_total_time": 65077.63999271393, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058499247856445, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1449286964354406, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2856, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2833, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9435084241823588, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9456953642384106, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9446006284107822, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054304635761589407, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9359101420548398, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9380794701986755, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9369935505209195, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9435084241823588, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058927537358204, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13364913333157613, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2135, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2129, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9438549955791335, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514260249554367, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9476253883710608, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04857397504456328, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9412024756852343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9487522281639929, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9449622725255216, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9438549955791335, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013497977994631, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13412408545720433, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1190, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1181, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9239130434782609, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9333333333333333, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9285992976980102, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06666666666666667, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9169254658385093, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9262745098039216, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9215762777994537, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9239130434782609} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_22000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..e6fa5aa44c70ecd6812f1978cb433a73aedd962f --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0422588592003033, "train": {"Classification_binary": 0.21010428667068481, "Cardinality": 0.18143826723098755, "Classification": 0.10037446022033691, "Regression_pt": 0.14237573742866516, "Regression_eta": 0.0018071241211146116, "Regression_sin_phi": 0.001001318683847785, "Regression_cos_phi": 0.001092968974262476, "Regression_energy": 0.1387682408094406, "HitGrouping": 0.16781064867973328, "Auxiliary": 0.25022053718566895, "Total": 1.2255712747573853, "Total_unweighted": 0.5955241369083524}, "train_group": {}, "valid": {"Classification_binary": 0.18151116371154785, "Cardinality": 0.13350695371627808, "Classification": 0.09355568885803223, "Regression_pt": 0.14106225967407227, "Regression_eta": 0.0018204205359021823, "Regression_sin_phi": 0.000989227555692196, "Regression_cos_phi": 0.0010367377350727718, "Regression_energy": 0.13988125324249268, "HitGrouping": 0.16548879941304526, "Auxiliary": 0.22177465756734213, "Total": 1.1093697547912598, "Total_unweighted": 0.5598567513128122, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8426385521888733, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.557054340839386, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3259658217430115, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5619117021560669, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5561085939407349, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.347501665353775, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7895588278770447, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.517952024936676, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34700101613998413, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5239756107330322, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5203102231025696, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.44357845187187195, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8530778288841248, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5451236963272095, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.30472058057785034, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5477026700973511, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5376519560813904, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33581241965293884, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8440211415290833, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5465257167816162, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3153495490550995, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5501468181610107, "metrics/hit_grouping/calorimeter/all/object_purity": 0.542035698890686, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34924012422561646, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026081029325723648, "metrics/hit_grouping/calorimeter/background_recall": 0.6401070952415466, "metrics/hit_grouping/calorimeter/completeness": 0.6129056811332703, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6180841326713562, "metrics/hit_grouping/calorimeter/energy_purity": 0.6109170913696289, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3575041592121124, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8633633852005005, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5639519095420837, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2482532113790512, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.571205735206604, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5598121285438538, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3009853661060333, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8463975191116333, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.564033031463623, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3272683024406433, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5659408569335938, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5589948296546936, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.35116448998451233, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8006036877632141, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4913959205150604, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44495007395744324, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4891132712364197, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4867359399795532, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4493742883205414, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8300625681877136, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5709424614906311, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3729056417942047, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5766271948814392, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5777040123939514, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.38766738772392273, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8002796173095703, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6448773145675659, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4918890595436096, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6411952972412109, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6552019119262695, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4317111372947693, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8646247982978821, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5076907277107239, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23088255524635315, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5100278258323669, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4892089068889618, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2973751127719879, "metrics/hit_grouping/calorimeter/purity": 0.6040594577789307, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8391868472099304, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.14300595223903656, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13384614884853363, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1441471427679062, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.09424538910388947, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8403928279876709, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3153865933418274, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1294184774160385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.31426697969436646, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.29795336723327637, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24116306006908417, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8443328738212585, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5684205889701843, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32991212606430054, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5723671913146973, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5656185746192932, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.36191630363464355, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7310663461685181, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4124327600002289, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5021860003471375, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5400300621986389, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5863717794418335, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7949585318565369, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3140774071216583, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5544046759605408, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5562207102775574, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4318564236164093, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6629273295402527, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43317317962646484, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4184432327747345, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45205992460250854, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6807119846343994, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6955836415290833, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4165879487991333, "metrics/hit_grouping/tracker/all/object_completeness": 0.45636242628097534, "metrics/hit_grouping/tracker/all/object_purity": 0.488637238740921, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6299357414245605, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005871462170034647, "metrics/hit_grouping/tracker/background_recall": 0.04874726012349129, "metrics/hit_grouping/tracker/completeness": 0.47339969873428345, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45677506923675537, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7306111454963684, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4044646918773651, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5372375249862671, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5549342632293701, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5846013426780701, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6671310663223267, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4309918284416199, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4173193871974945, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4680825173854828, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6773272156715393, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6480582356452942, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4281958043575287, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3183141350746155, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.36330646276474, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6802184581756592, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6989179849624634, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42316678166389465, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.45522716641426086, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4987022578716278, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6199676394462585, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6823942065238953, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.43275633454322815, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4363521635532379, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5063768029212952, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6471371650695801, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6935590505599976, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4051744341850281, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.46135997772216797, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.47240591049194336, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6398969888687134, "metrics/hit_grouping/tracker/purity": 0.4516739547252655, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.883687436580658, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04300608113408089, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11549516767263412, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11821487545967102, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07515203952789307, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6794142127037048, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2545360326766968, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3317604660987854, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.34504374861717224, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5629860162734985, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6718645691871643, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48580366373062134, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5168685913085938, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5553218126296997, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7129449248313904, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5365630388259888, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6460986137390137, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7276065349578857, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6427880525588989, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05083564296364784, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1119832918047905, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.116336822509766, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3055257797241211, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17508596181869507, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8621092438697815, "metrics/particle/class_1/pid_accuracy": 0.9658488631248474, "metrics/particle/class_2/efficiency": 0.49203500151634216, "metrics/particle/class_2/pid_accuracy": 0.6894664764404297, "metrics/particle/class_3/efficiency": 0.7511852979660034, "metrics/particle/class_3/pid_accuracy": 0.9640439748764038, "metrics/particle/class_4/efficiency": 0.6794995069503784, "metrics/particle/class_4/pid_accuracy": 0.44192636013031006, "metrics/particle/class_5/efficiency": 0.8880000114440918, "metrics/particle/class_5/pid_accuracy": 0.5765765905380249, "metrics/particle/count/bias_mean": -1.2037760019302368, "metrics/particle/count/mae": 3.7649738788604736, "metrics/particle/count/prediction_mean": 68.85221099853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.07419098168611526, "metrics/particle/event/energy_response_mean": 0.9862310290336609, "metrics/particle/event/met_abs_error": 7.291969299316406, "metrics/particle/event/scalar_pt_relative_abs_error": 0.058596014976501465, "metrics/particle/event/scalar_pt_response_mean": 0.9991434216499329, "metrics/particle/event/vector_pt_closure": 0.059863921254873276, "metrics/particle/matched/delta_eta_abs_mean": 0.016469229012727737, "metrics/particle/matched/delta_phi_abs_mean": 0.01647826097905636, "metrics/particle/matched/delta_r_mean": 0.026137784123420715, "metrics/particle/matched/energy_relative_abs_mean": 0.15014277398586273, "metrics/particle/matched/pid_accuracy": 0.9442357420921326, "metrics/particle/matched/pt_relative_abs_mean": 0.1494869440793991, "metrics/particle/matching/duplicate_fraction": 0.06280435621738434, "metrics/particle/matching/efficiency": 0.7817593812942505, "metrics/particle/matching/f1": 0.7885341048240662, "metrics/particle/matching/purity": 0.7954272627830505, "metrics/particle/matching/target_coverage_dr0p05": 0.7041057348251343, "metrics/particle/matching/target_coverage_dr0p10": 0.8280857801437378, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5807761549949646, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8280393481254578, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02506438083946705, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14539533853530884, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14603786170482635, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019453080371022224, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1864147037267685, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007590132765471935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7436699867248535, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.019930263981223106, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.022109031677246, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8432.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2831445038318634, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4653905630111694, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.7110719680786133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5492591857910156, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14215686917304993, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 49.94490051269531, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01605151779949665, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0810725688934326, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.010249800980091095, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0920519828796387, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10404.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7710314393043518, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.08185239136219025, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.841133117675781, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.938462972640991, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.03674808889627457, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7701506614685059, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.544680118560791, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5441104769706726, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.604230880737305, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023564917966723442, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1436038613319397, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15069624781608582, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03674251586198807, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1911909580230713, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023970818147063255, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4431644678115845, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.04323054850101471, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9113669395446777, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1919.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5127388834953308, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9721444845199585, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.4469103813171387, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2897799015045166, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20911528170108795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.225332260131836, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.057066258043050766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5063107013702393, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05002463236451149, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.7955098152160645, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2611.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6352201104164124, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.099851608276367, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.57254409790039, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2863214015960693, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05872197076678276, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.2080668807029724, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.440886497497559, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6522037982940674, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.50032615661621, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026510894298553467, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15258559584617615, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14991341531276703, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014047524891793728, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18956118822097778, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009274287149310112, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.25791049003601074, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.01398236770182848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8147332668304443, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17252.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22692738473415375, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1069777011871338, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8515998721122742, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6733644604682922, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13160276412963867, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.316353797912598, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004572517238557339, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4580302834510803, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.013427110388875008, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0120513439178467, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20995.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8047767877578735, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7473047971725464, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.496456146240234, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.083090782165527, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6365086436271667, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.9584851264953613, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.066592693328857, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.028554603457450867, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.168058395385742, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025864213705062866, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14976473152637482, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14878396689891815, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017276568338274956, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18871991336345673, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0045647211372852325, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.48869800567626953, "metrics/particle/momentum/all/accepted/pz_bias": -0.011821771040558815, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6515223979949951, "metrics/particle/momentum/all/accepted/samples": 27603.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26606693863868713, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.206268548965454, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2369954586029053, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0654041767120361, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14078211784362793, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 29.169158935546875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0121140843257308, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8058748245239258, "metrics/particle/momentum/all/assigned/pz_bias": -0.008993753232061863, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1763603687286377, "metrics/particle/momentum/all/assigned/samples": 34010.0, "metrics/particle/momentum/all/efficiency": 0.7798779606819153, "metrics/particle/momentum/all/event/energy_bias": -3.434398651123047, "metrics/particle/momentum/all/event/p3_error_mean": 20.92755889892578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.016621589660645, "metrics/particle/momentum/all/event/px_bias": -0.7319787740707397, "metrics/particle/momentum/all/event/py_bias": 2.9367024898529053, "metrics/particle/momentum/all/event/pz_abs_mean": 15.629066467285156, "metrics/particle/momentum/all/event/pz_bias": -1.2248687744140625, "metrics/particle/momentum/all/event/pz_residual_rms": 25.715600967407227, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02822117879986763, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13733865320682526, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13578425347805023, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.031356729567050934, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17399479448795319, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003822630038484931, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5005647540092468, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.003645107615739107, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4755988121032715, "metrics/particle/momentum/class_1/accepted/samples": 14388.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1467002034187317, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3613037168979645, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3650123178958893, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.22687853872776031, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08037897199392319, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9028844833374023, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008557457476854324, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.686148464679718, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.022086631506681442, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6841397285461426, "metrics/particle/momentum/class_1/assigned/samples": 16360.0, "metrics/particle/momentum/class_1/efficiency": 0.8622280955314636, "metrics/particle/momentum/class_1/event/energy_bias": 9.09953498840332, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.29457664489746, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.72844123840332, "metrics/particle/momentum/class_1/event/px_bias": -0.3366398811340332, "metrics/particle/momentum/class_1/event/py_bias": 0.6566512584686279, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.521989822387695, "metrics/particle/momentum/class_1/event/pz_bias": 1.288461685180664, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.68730354309082, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03043595887720585, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19034169614315033, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19157220423221588, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.031060725450515747, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2328891009092331, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02389078587293625, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5510915517807007, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.020912641659379005, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.85935115814209, "metrics/particle/momentum/class_2/accepted/samples": 1465.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6774747967720032, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7119673490524292, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8539385199546814, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4741283655166626, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2890269160270691, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.7233364582061768, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.053002070635557175, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.518995761871338, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07057254761457443, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.260546684265137, "metrics/particle/momentum/class_2/assigned/samples": 2415.0, "metrics/particle/momentum/class_2/efficiency": 0.4921061396598816, "metrics/particle/momentum/class_2/event/energy_bias": -14.447978019714355, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.978858947753906, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.701821327209473, "metrics/particle/momentum/class_2/event/px_bias": 0.8899930715560913, "metrics/particle/momentum/class_2/event/py_bias": -0.020692676305770874, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.197517395019531, "metrics/particle/momentum/class_2/event/pz_bias": -0.1410614252090454, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.737686157226562, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02237841859459877, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15933921933174133, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15895119309425354, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.004164120182394981, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19912898540496826, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002727912738919258, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3049725294113159, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04204748943448067, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2581827640533447, "metrics/particle/momentum/class_3/accepted/samples": 11364.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3299172818660736, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.220874786376953, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.275207281112671, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.099409818649292, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18178731203079224, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 44.18653869628906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00896373763680458, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6306303143501282, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.057499177753925323, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.88424015045166, "metrics/particle/momentum/class_3/assigned/samples": 14726.0, "metrics/particle/momentum/class_3/efficiency": 0.7472874522209167, "metrics/particle/momentum/class_3/event/energy_bias": 2.362741708755493, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.476616859436035, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.32299280166626, "metrics/particle/momentum/class_3/event/px_bias": -1.2131143808364868, "metrics/particle/momentum/class_3/event/py_bias": 2.45969820022583, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.116724967956543, "metrics/particle/momentum/class_3/event/pz_bias": -2.9043631553649902, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.57944107055664, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025568382814526558, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1788327693939209, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1723715364933014, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08495303243398666, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21349580585956573, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004065040498971939, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0860329866409302, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4250534474849701, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6916089057922363, "metrics/particle/momentum/class_4/accepted/samples": 246.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.39170247316360474, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.851541519165039, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4068472385406494, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2881312370300293, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24011299014091492, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.273517608642578, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2792338132858276, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08518525958061218, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.7088916301727295, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.6758241653442383, "metrics/particle/momentum/class_4/event/energy_bias": -0.4360125660896301, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.913210868835449, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0058815479278564, "metrics/particle/momentum/class_4/event/px_bias": -0.08365555107593536, "metrics/particle/momentum/class_4/event/py_bias": -0.11422258615493774, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7764344215393066, "metrics/particle/momentum/class_4/event/pz_bias": 0.5218005180358887, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.783816814422607, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01926184631884098, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17395302653312683, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17029806971549988, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02149256318807602, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2108120173215866, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02857142873108387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.015592098236084, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.17956693470478058, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.1121134757995605, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.1019073948264122, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2245885580778122, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.21696282923221588, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.036981694400310516, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08387096971273422, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.29496604204177856, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04516129195690155, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.3195273876190186, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.13593673706054688, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.572837829589844, "metrics/particle/momentum/class_5/assigned/samples": 155.0, "metrics/particle/momentum/class_5/efficiency": 0.8805031180381775, "metrics/particle/momentum/class_5/event/energy_bias": -0.012684516608715057, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.4110283851623535, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8901569843292236, "metrics/particle/momentum/class_5/event/px_bias": 0.011438049376010895, "metrics/particle/momentum/class_5/event/py_bias": -0.044731512665748596, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.208350658416748, "metrics/particle/momentum/class_5/event/pz_bias": 0.010293610394001007, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.708958625793457, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3918973207473755, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3940909802913666, "metrics/set_slot/angular_recall_all_dr0p10": 0.9418619871139526, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8126405477523804, "metrics/set_slot/soft_count_bias_mean": 0.7062559723854065}, "step_train_time": 6.236901521682739, "step_valid_time": 68178.7066192627, "step_total_time": 68184.9435210228, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066886550325802, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16153413792295146, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2872, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2844, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948794185662372, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9462932454695222, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9475420653249752, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05370675453047776, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9395441030723488, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9370675453047776, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9383041900362916, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948794185662372, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9966749218024068, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1444508014584598, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2131, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9420866489832007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9483756119270138, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9452206697715679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0516243880729862, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9385499557913352, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9448153093012907, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.941672211133289, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9420866489832007, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9964907894715878, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14940502051268245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1276, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1188, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1185, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.922360248447205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9310344827586207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9266770670826833, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06896551724137931, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9200310559006211, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9286833855799373, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9243369734789392, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.922360248447205} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_23000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..e015fbef6e966f9f5fa9fe3450290549f79c5548 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.089634261891226, "train": {"Classification_binary": 0.1978994905948639, "Cardinality": 0.14700880646705627, "Classification": 0.08176812529563904, "Regression_pt": 0.12927499413490295, "Regression_eta": 0.001904753502458334, "Regression_sin_phi": 0.0010755376424640417, "Regression_cos_phi": 0.0009649453568272293, "Regression_energy": 0.12821745872497559, "HitGrouping": 0.16426236927509308, "Auxiliary": 0.22343631088733673, "Total": 1.1236076354980469, "Total_unweighted": 0.5411053052521311}, "train_group": {}, "valid": {"Classification_binary": 0.17602801322937012, "Cardinality": 0.12181089321772258, "Classification": 0.08052706221739452, "Regression_pt": 0.13036475578943887, "Regression_eta": 0.001771357065687577, "Regression_sin_phi": 0.0009707011437664429, "Regression_cos_phi": 0.0010065547345827024, "Regression_energy": 0.13040608167648315, "HitGrouping": 0.16134055455525717, "Auxiliary": 0.2120213508605957, "Total": 1.0548516114552815, "Total_unweighted": 0.5210745258567234, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8414740562438965, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5641087889671326, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.31075847148895264, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5693308115005493, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5661565661430359, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.35534146428108215, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7904659509658813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5250788331031799, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.31548628211021423, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5323516130447388, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5374880433082581, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4293459951877594, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8520709276199341, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5465921759605408, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.29718005657196045, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5496751666069031, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5459205508232117, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3391178548336029, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8431351780891418, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5502992868423462, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.30319148302078247, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5544463396072388, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5516760349273682, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.35243162512779236, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026366276666522026, "metrics/hit_grouping/calorimeter/background_recall": 0.6439071893692017, "metrics/hit_grouping/calorimeter/completeness": 0.6247122287750244, "metrics/hit_grouping/calorimeter/energy_completeness": 0.634777307510376, "metrics/hit_grouping/calorimeter/energy_purity": 0.6276260018348694, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3505840301513672, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8620309829711914, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5640685558319092, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24365481734275818, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5708033442497253, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5591762065887451, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3017617166042328, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8447224497795105, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5740804076194763, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.31477439403533936, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5781615972518921, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5783790946006775, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3605045974254608, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8014847636222839, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4963718056678772, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4171406924724579, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.49511057138442993, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5079732537269592, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.45127037167549133, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8288449645042419, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5819781422615051, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.35459357500076294, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.588334321975708, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6000387072563171, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3972594141960144, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.82198566198349, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.664270281791687, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.46572476625442505, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6633143424987793, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6673290133476257, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37938252091407776, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8612045049667358, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5016582608222961, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22713272273540497, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5040835738182068, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48488369584083557, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.30079013109207153, "metrics/hit_grouping/calorimeter/purity": 0.6158759593963623, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8178223371505737, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1239946112036705, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12153846025466919, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1233699768781662, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08358337730169296, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04923076927661896, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8342718482017517, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.28313466906547546, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1288483440876007, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28108036518096924, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2720530927181244, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24116306006908417, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8441844582557678, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5747517943382263, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.317090779542923, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.579357385635376, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5777357220649719, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.36529380083084106, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7298329472541809, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41063958406448364, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5053666234016418, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5418272614479065, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5917513370513916, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.798284113407135, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32024678587913513, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5533064007759094, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5585260987281799, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42512619495391846, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6607292294502258, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42553359270095825, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4205487072467804, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4677140712738037, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6819721460342407, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6941555142402649, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4119592010974884, "metrics/hit_grouping/tracker/all/object_completeness": 0.4585379660129547, "metrics/hit_grouping/tracker/all/object_purity": 0.49878737330436707, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6318250298500061, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00469716964289546, "metrics/hit_grouping/tracker/background_recall": 0.03328792378306389, "metrics/hit_grouping/tracker/completeness": 0.4741361141204834, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45125675201416016, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7283540368080139, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3949885964393616, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5374269485473633, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5635831356048584, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5926195979118347, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6678810119628906, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43784505128860474, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4231921434402466, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4806500971317291, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6702836751937866, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6461563110351562, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4221278429031372, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32097017765045166, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.37422046065330505, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6779935359954834, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6946505308151245, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4179547131061554, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.45473524928092957, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5110490918159485, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6287742853164673, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6948434114456177, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4380825459957123, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45314493775367737, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.514070987701416, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6058588624000549, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6933898329734802, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39955514669418335, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4644402265548706, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4801260530948639, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6403652429580688, "metrics/hit_grouping/tracker/purity": 0.4518428444862366, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8881871700286865, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04517810419201851, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13087761402130127, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13347336649894714, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07471763342618942, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6814799904823303, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26645931601524353, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3318116068840027, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3388594090938568, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5515811443328857, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6692332625389099, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47836610674858093, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.517491340637207, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5666378140449524, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7166637182235718, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5494974255561829, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6712634563446045, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7595018148422241, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6509560346603394, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0637701004743576, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12015131860971451, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.946935653686523, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2921796143054962, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1617397964000702, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.870201826095581, "metrics/particle/class_1/pid_accuracy": 0.9687585234642029, "metrics/particle/class_2/efficiency": 0.509535551071167, "metrics/particle/class_2/pid_accuracy": 0.7371202111244202, "metrics/particle/class_3/efficiency": 0.7696350812911987, "metrics/particle/class_3/pid_accuracy": 0.9677491188049316, "metrics/particle/class_4/efficiency": 0.7054860591888428, "metrics/particle/class_4/pid_accuracy": 0.46930423378944397, "metrics/particle/class_5/efficiency": 0.8799999952316284, "metrics/particle/class_5/pid_accuracy": 0.6136363744735718, "metrics/particle/count/bias_mean": -1.3333333730697632, "metrics/particle/count/mae": 3.68359375, "metrics/particle/count/prediction_mean": 68.72265625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06695236265659332, "metrics/particle/event/energy_response_mean": 0.9873855113983154, "metrics/particle/event/met_abs_error": 6.584871768951416, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05589296296238899, "metrics/particle/event/scalar_pt_response_mean": 0.9959967732429504, "metrics/particle/event/vector_pt_closure": 0.05496169626712799, "metrics/particle/matched/delta_eta_abs_mean": 0.01643150858581066, "metrics/particle/matched/delta_phi_abs_mean": 0.01694614626467228, "metrics/particle/matched/delta_r_mean": 0.026436341926455498, "metrics/particle/matched/energy_relative_abs_mean": 0.14470218122005463, "metrics/particle/matched/pid_accuracy": 0.9499357342720032, "metrics/particle/matched/pt_relative_abs_mean": 0.14408329129219055, "metrics/particle/matching/duplicate_fraction": 0.05973019450902939, "metrics/particle/matching/efficiency": 0.795215904712677, "metrics/particle/matching/f1": 0.8028560280799866, "metrics/particle/matching/purity": 0.8106443881988525, "metrics/particle/matching/target_coverage_dr0p05": 0.7102392315864563, "metrics/particle/matching/target_coverage_dr0p10": 0.8352693915367126, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6054680943489075, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8402784466743469, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025944465771317482, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1449631154537201, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14524763822555542, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02302130125463009, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1827840954065323, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006685108412057161, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7024253606796265, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.09613881260156631, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8332862854003906, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8676.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2628234922885895, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0825923681259155, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2536523342132568, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.058302402496338, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12461715191602707, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 22.37023162841797, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01741960272192955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0879952907562256, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.1286984086036682, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.262284278869629, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10448.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7933430671691895, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5851512551307678, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.23624324798584, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.110533714294434, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24737337231636047, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3375179171562195, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.938455581665039, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.2608513832092285, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.748659133911133, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024369200691580772, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1566920429468155, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15238669514656067, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.030807871371507645, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19010956585407257, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02962603233754635, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5160386562347412, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1819763332605362, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.974977970123291, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2059.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4392426609992981, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.3153886795043945, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.965940237045288, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.760740280151367, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1614859700202942, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 105.16696166992188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05534495785832405, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3558599948883057, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0464165024459362, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.079904079437256, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2638.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6815624237060547, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.744149684906006, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.274869918823242, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2434626817703247, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05451750010251999, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.2564898431301117, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.131478309631348, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3925125300884247, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.823135375976562, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026658719405531883, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14277192950248718, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14260774850845337, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0073792533949017525, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1814243495464325, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000802200345788151, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2479916661977768, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.035730455070734024, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7464261054992676, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17452.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22856910526752472, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.222350001335144, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9286549687385559, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7409471869468689, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12421563267707825, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.289005279541016, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00617988221347332, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4751442074775696, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.028127120807766914, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0513381958007812, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21036.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8141064643859863, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.2719656229019165, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.97580623626709, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.579451560974121, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5788286924362183, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.39110904932022095, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.065923690795898, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.277097225189209, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.603612899780273, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026271626353263855, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1444632112979889, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14413462579250336, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.01390530914068222, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18249092996120453, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004718487150967121, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4804951548576355, "metrics/particle/momentum/all/accepted/pz_bias": 0.06500718742609024, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5917447805404663, "metrics/particle/momentum/all/accepted/samples": 28187.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2553449869155884, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2640604972839355, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.262982726097107, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.071582555770874, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12721997499465942, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 32.41374588012695, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013422425836324692, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8081960678100586, "metrics/particle/momentum/all/assigned/pz_bias": 0.060335561633110046, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9513888359069824, "metrics/particle/momentum/all/assigned/samples": 34122.0, "metrics/particle/momentum/all/efficiency": 0.7963778972625732, "metrics/particle/momentum/all/event/energy_bias": -1.8870329856872559, "metrics/particle/momentum/all/event/p3_error_mean": 19.485633850097656, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.628730773925781, "metrics/particle/momentum/all/event/px_bias": -0.8807195425033569, "metrics/particle/momentum/all/event/py_bias": -0.985116720199585, "metrics/particle/momentum/all/event/pz_abs_mean": 14.638340950012207, "metrics/particle/momentum/all/event/pz_bias": 3.9304609298706055, "metrics/particle/momentum/all/event/pz_residual_rms": 21.157922744750977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029507216066122055, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1290687471628189, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1294027417898178, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0074240053072571754, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16543255746364594, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004182952921837568, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4959541857242584, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.113644540309906, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5203498601913452, "metrics/particle/momentum/class_1/accepted/samples": 14583.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14886920154094696, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3606838285923004, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.377160906791687, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20939844846725464, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0707656592130661, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.744076728820801, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008548052050173283, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6728968620300293, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.12382794171571732, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2917144298553467, "metrics/particle/momentum/class_1/assigned/samples": 16378.0, "metrics/particle/momentum/class_1/efficiency": 0.8739138245582581, "metrics/particle/momentum/class_1/event/energy_bias": 12.059185981750488, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.047927856445312, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.539719581604004, "metrics/particle/momentum/class_1/event/px_bias": 0.06467965245246887, "metrics/particle/momentum/class_1/event/py_bias": -1.0481359958648682, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.335824966430664, "metrics/particle/momentum/class_1/event/pz_bias": 5.175220966339111, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.814653396606445, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029491057619452477, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1857137680053711, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18715587258338928, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.030379706993699074, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22678671777248383, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024716099724173546, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6126446723937988, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06785751134157181, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.849170684814453, "metrics/particle/momentum/class_2/accepted/samples": 1497.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6643396615982056, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8541185855865479, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.12083899974823, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7469668388366699, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27941176295280457, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.266280174255371, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05637254938483238, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.656378746032715, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0701073631644249, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.583317756652832, "metrics/particle/momentum/class_2/assigned/samples": 2448.0, "metrics/particle/momentum/class_2/efficiency": 0.5028552412986755, "metrics/particle/momentum/class_2/event/energy_bias": -12.032073020935059, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.191322326660156, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.217272758483887, "metrics/particle/momentum/class_2/event/px_bias": 0.3988392949104309, "metrics/particle/momentum/class_2/event/py_bias": -0.4418463706970215, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.607651710510254, "metrics/particle/momentum/class_2/event/pz_bias": -2.184293270111084, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.72871971130371, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02182687632739544, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15812821686267853, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15666131675243378, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.02159636840224266, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.195389986038208, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0023084816057235003, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2894372344017029, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.010041850619018078, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.017311930656433, "metrics/particle/momentum/class_3/accepted/samples": 11696.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3061441481113434, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.3155200481414795, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2657806873321533, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.076528310775757, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16336332261562347, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 48.97755432128906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011093824170529842, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6276711225509644, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0056385863572359085, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5695443153381348, "metrics/particle/momentum/class_3/assigned/samples": 14783.0, "metrics/particle/momentum/class_3/efficiency": 0.7691195011138916, "metrics/particle/momentum/class_3/event/energy_bias": -1.5892341136932373, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.921234130859375, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.2926130294799805, "metrics/particle/momentum/class_3/event/px_bias": -0.9880148768424988, "metrics/particle/momentum/class_3/event/py_bias": 0.18066057562828064, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.188757419586182, "metrics/particle/momentum/class_3/event/pz_bias": -0.18043716251850128, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.918722152709961, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02930644527077675, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15075092017650604, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1506824642419815, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02311466634273529, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19371084868907928, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.014814814552664757, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8698475956916809, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4915328323841095, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.197528839111328, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3250475525856018, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.443333864212036, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8120195865631104, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6634206771850586, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19327731430530548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.165462493896484, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0873843431472778, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.24319897592067719, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2021961212158203, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.7417582273483276, "metrics/particle/momentum/class_4/event/energy_bias": -1.2334401607513428, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6788740158081055, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7915730476379395, "metrics/particle/momentum/class_4/event/px_bias": -0.14535430073738098, "metrics/particle/momentum/class_4/event/py_bias": 0.028808223083615303, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6463892459869385, "metrics/particle/momentum/class_4/event/pz_bias": 0.7856954336166382, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.14180326461792, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020330162718892097, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15312810242176056, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15940235555171967, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05775841325521469, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20554280281066895, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.028368793427944183, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9643337726593018, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.18793275952339172, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.8774807453155518, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.042488742619752884, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20199783146381378, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20907551050186157, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.09796550124883652, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2957492470741272, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.057692307978868484, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.478776693344116, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.20616364479064941, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.965909957885742, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8867924809455872, "metrics/particle/momentum/class_5/event/energy_bias": 0.9085283279418945, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.755141019821167, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.078148126602173, "metrics/particle/momentum/class_5/event/px_bias": -0.21086923778057098, "metrics/particle/momentum/class_5/event/py_bias": 0.2953968048095703, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.39512038230896, "metrics/particle/momentum/class_5/event/pz_bias": 0.3342752456665039, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.534011363983154, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39114242792129517, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.39264506101608276, "metrics/set_slot/angular_recall_all_dr0p10": 0.9451424479484558, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8212181329727173, "metrics/set_slot/soft_count_bias_mean": 0.8153801560401917}, "step_train_time": 6.277699947357178, "step_valid_time": 71341.52624893188, "step_total_time": 71347.80394935608, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005563262396129, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14631887481602157, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3003, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2859, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2826, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9444995044598612, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952047952047952, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9482587064676616, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04795204795204795, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.933597621407334, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9410589410589411, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9373134328358209, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9444995044598612, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9926030101604154, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1443582102906149, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2127, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9403183023872679, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9525302283922974, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.946384872080089, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04746977160770264, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9385499557913352, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9507389162561576, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9446051167964404, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9403183023872679, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9936443918940587, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13753698412057958, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1274, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1189, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9231366459627329, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9332810047095761, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9281811085089774, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06671899529042387, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9208074534161491, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9309262166405023, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9258391881342701, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9231366459627329} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_24000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..35df1330b525935917c1e3fd40dfd4de971a1516 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.137009664582149, "train": {"Classification_binary": 0.16708609461784363, "Cardinality": 0.12205128371715546, "Classification": 0.0862765908241272, "Regression_pt": 0.12056957930326462, "Regression_eta": 0.001524380873888731, "Regression_sin_phi": 0.0009967051446437836, "Regression_cos_phi": 0.0009844054002314806, "Regression_energy": 0.11989200115203857, "HitGrouping": 0.15519581735134125, "Auxiliary": 0.20757296681404114, "Total": 1.02140212059021, "Total_unweighted": 0.497329757316038}, "train_group": {}, "valid": {"Classification_binary": 0.16782087087631226, "Cardinality": 0.1146086057027181, "Classification": 0.08082477748394012, "Regression_pt": 0.12282305955886841, "Regression_eta": 0.0016659082223971684, "Regression_sin_phi": 0.0009218489285558462, "Regression_cos_phi": 0.0009561933887501558, "Regression_energy": 0.12262938419977824, "HitGrouping": 0.15759113430976868, "Auxiliary": 0.2045374313990275, "Total": 1.0098645687103271, "Total_unweighted": 0.4976420426586022, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.844504177570343, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.566827654838562, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.31737035512924194, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5733197331428528, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5673903226852417, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3446680009365082, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7951434254646301, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5275929570198059, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34937310218811035, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5343246459960938, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.525307297706604, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4232463538646698, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8568353652954102, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5552805662155151, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.30275797843933105, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.55787193775177, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5466756820678711, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31975001096725464, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8473337888717651, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5565128922462463, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3116413354873657, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5607308149337769, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5514249205589294, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.337051659822464, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026246679946780205, "metrics/hit_grouping/calorimeter/background_recall": 0.6431668996810913, "metrics/hit_grouping/calorimeter/completeness": 0.6290395855903625, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6390342712402344, "metrics/hit_grouping/calorimeter/energy_purity": 0.6316213011741638, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3544512987136841, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.864540696144104, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5651772618293762, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25273215770721436, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.572557270526886, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.556290864944458, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.29065391421318054, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8503321409225464, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5797039866447449, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3192625045776367, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5832688808441162, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5736821889877319, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33830663561820984, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8077877163887024, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5140057802200317, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.42839086055755615, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5122114419937134, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5179309248924255, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4339527189731598, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8342281579971313, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5865588188171387, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.35870444774627686, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5928753614425659, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5990062355995178, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3783245086669922, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8193225860595703, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6752022504806519, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.46677133440971375, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.672407865524292, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6706811189651489, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3778126537799835, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8650073409080505, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5090236663818359, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24119459092617035, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5118829011917114, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4850114583969116, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28746485710144043, "metrics/hit_grouping/calorimeter/purity": 0.6200659275054932, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.826998770236969, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.14183375239372253, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13846154510974884, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.14044322073459625, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0969947949051857, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03384615480899811, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8323608636856079, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3146827816963196, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1277080923318863, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.31345292925834656, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3001343607902527, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2366020530462265, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8486283421516418, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.579260528087616, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32591158151626587, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5839113593101501, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.575563907623291, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34929171204566956, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7364825010299683, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4193066358566284, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49928611516952515, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5277817845344543, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5697848200798035, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7980377674102783, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34492427110671997, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.560481071472168, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5475977063179016, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42063936591148376, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6712639927864075, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4270299971103668, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42233410477638245, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45733335614204407, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6675592660903931, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7025543451309204, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4176742732524872, "metrics/hit_grouping/tracker/all/object_completeness": 0.4582909643650055, "metrics/hit_grouping/tracker/all/object_purity": 0.4872021973133087, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6158605813980103, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00696035148575902, "metrics/hit_grouping/tracker/background_recall": 0.05188651382923126, "metrics/hit_grouping/tracker/completeness": 0.472907692193985, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46412476897239685, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7301180958747864, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40154898166656494, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5293334722518921, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5474728345870972, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.580227792263031, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6849673390388489, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4361317455768585, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4276573956012726, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.46791380643844604, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6440129280090332, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6600529551506042, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4338592290878296, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3331345021724701, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3739035725593567, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6650485396385193, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7100218534469604, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4259525537490845, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4581044316291809, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.49400416016578674, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6032530665397644, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6988543272018433, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4567243754863739, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4573593735694885, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.508523166179657, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6245006918907166, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6934767961502075, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4000234007835388, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.45869770646095276, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.474592000246048, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6307656168937683, "metrics/hit_grouping/tracker/purity": 0.45169928669929504, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8881710767745972, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05343179777264595, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13476446270942688, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13215522468090057, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07862728089094162, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6861106157302856, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2768273651599884, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.34206444025039673, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3484959900379181, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5432866811752319, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6792045831680298, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4832064211368561, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5154870748519897, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5512410402297974, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6971253156661987, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5656444430351257, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6851754188537598, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7736912965774536, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6445448994636536, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.07991708815097809, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.11374016106128693, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.92339515686035, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28311774134635925, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15267793834209442, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8846853971481323, "metrics/particle/class_1/pid_accuracy": 0.9718419909477234, "metrics/particle/class_2/efficiency": 0.5157056450843811, "metrics/particle/class_2/pid_accuracy": 0.7174243927001953, "metrics/particle/class_3/efficiency": 0.764614999294281, "metrics/particle/class_3/pid_accuracy": 0.9708482027053833, "metrics/particle/class_4/efficiency": 0.7362849116325378, "metrics/particle/class_4/pid_accuracy": 0.4601307213306427, "metrics/particle/class_5/efficiency": 0.9359999895095825, "metrics/particle/class_5/pid_accuracy": 0.6367521286010742, "metrics/particle/count/bias_mean": -1.6484375, "metrics/particle/count/mae": 3.6614582538604736, "metrics/particle/count/prediction_mean": 68.40755462646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06228584423661232, "metrics/particle/event/energy_response_mean": 0.9890996813774109, "metrics/particle/event/met_abs_error": 6.671482563018799, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05055220425128937, "metrics/particle/event/scalar_pt_response_mean": 1.003159523010254, "metrics/particle/event/vector_pt_closure": 0.051719073206186295, "metrics/particle/matched/delta_eta_abs_mean": 0.015570097602903843, "metrics/particle/matched/delta_phi_abs_mean": 0.01579045131802559, "metrics/particle/matched/delta_r_mean": 0.02487477846443653, "metrics/particle/matched/energy_relative_abs_mean": 0.1370471715927124, "metrics/particle/matched/pid_accuracy": 0.9514985680580139, "metrics/particle/matched/pt_relative_abs_mean": 0.13654668629169464, "metrics/particle/matching/duplicate_fraction": 0.0570550262928009, "metrics/particle/matching/efficiency": 0.8009125590324402, "metrics/particle/matching/f1": 0.810447633266449, "metrics/particle/matching/purity": 0.8202124238014221, "metrics/particle/matching/target_coverage_dr0p05": 0.7303310036659241, "metrics/particle/matching/target_coverage_dr0p10": 0.842164933681488, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6263962984085083, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8422207236289978, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02393488772213459, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13470131158828735, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13394854962825775, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009058568626642227, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1734875589609146, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006901311222463846, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6677172780036926, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03349342569708824, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8334980010986328, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8694.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24775275588035583, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9640750885009766, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0249344110488892, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8703662157058716, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12051208317279816, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.626811981201172, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01607469469308853, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0287582874298096, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.012743216007947922, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2024052143096924, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10389.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7949890494346619, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8039730191230774, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.570876121520996, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8455235958099365, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2984311580657959, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09171532839536667, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.377729892730713, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6703121662139893, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.921027183532715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02405230887234211, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14215803146362305, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13969877362251282, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013819531537592411, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17985306680202484, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02551521174609661, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.344765067100525, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.02840542607009411, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.614666700363159, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2038.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.43917858600616455, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0850038528442383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3893728256225586, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2058162689208984, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.16888046264648438, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.57134246826172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05920303612947464, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.352247714996338, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1625935137271881, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.192564964294434, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2635.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6746110320091248, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.195085048675537, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.981773853302002, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1298853158950806, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.022541280835866928, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.12234392762184143, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.8512468338012695, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.1885430812835693, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.218515396118164, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02516063116490841, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13623592257499695, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1364024132490158, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01463163923472166, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17717088758945465, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008560666465200484, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23772914707660675, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03555166348814964, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7681885957717896, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17522.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21280644834041595, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1620991230010986, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8915566802024841, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7312136292457581, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11904079467058182, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.392060279846191, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004394764546304941, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4163717031478882, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05519405007362366, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6979492902755737, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20934.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8173718452453613, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5737320184707642, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.962164878845215, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.583380222320557, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2001299858093262, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3831560015678406, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9737727642059326, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.261824131011963, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.096312046051025, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024703513830900192, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13619087636470795, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1358851194381714, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012858178466558456, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17624177038669586, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004494938999414444, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.44989219307899475, "metrics/particle/momentum/all/accepted/pz_bias": -0.030305013060569763, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5306366682052612, "metrics/particle/momentum/all/accepted/samples": 28254.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24106334149837494, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.095534086227417, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0485862493515015, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8882085680961609, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12335826456546783, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.30732536315918, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012220978736877441, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.753939151763916, "metrics/particle/momentum/all/assigned/pz_bias": -0.050540532916784286, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8089916706085205, "metrics/particle/momentum/all/assigned/samples": 33958.0, "metrics/particle/momentum/all/efficiency": 0.7982708811759949, "metrics/particle/momentum/all/event/energy_bias": -2.8173797130584717, "metrics/particle/momentum/all/event/p3_error_mean": 18.627544403076172, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.278639793395996, "metrics/particle/momentum/all/event/px_bias": 1.4760198593139648, "metrics/particle/momentum/all/event/py_bias": 0.352527379989624, "metrics/particle/momentum/all/event/pz_abs_mean": 13.764391899108887, "metrics/particle/momentum/all/event/pz_bias": -5.120678901672363, "metrics/particle/momentum/all/event/pz_residual_rms": 21.490066528320312, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026741258800029755, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11654696613550186, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11654654145240784, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009802589192986488, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15401126444339752, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004334281664341688, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.46395403146743774, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.06349138170480728, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.456321358680725, "metrics/particle/momentum/class_1/accepted/samples": 14766.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12564310431480408, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3080509305000305, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2979530394077301, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1621088981628418, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.059511419385671616, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.68563175201416, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007959346286952496, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6001744866371155, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06531701236963272, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0224192142486572, "metrics/particle/momentum/class_1/assigned/samples": 16333.0, "metrics/particle/momentum/class_1/efficiency": 0.8848804235458374, "metrics/particle/momentum/class_1/event/energy_bias": 6.744280815124512, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.802687644958496, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.055428504943848, "metrics/particle/momentum/class_1/event/px_bias": 1.199516773223877, "metrics/particle/momentum/class_1/event/py_bias": -0.8504804372787476, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.97598934173584, "metrics/particle/momentum/class_1/event/pz_bias": -2.5817861557006836, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.3106689453125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03021608479321003, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18012087047100067, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1826777458190918, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.004377404227852821, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2228778898715973, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.018433179706335068, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3958147764205933, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1051306277513504, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.474515676498413, "metrics/particle/momentum/class_2/accepted/samples": 1519.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6158945560455322, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6890285611152649, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8202725648880005, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.46899664402008057, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26702216267585754, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.289470195770264, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05291222408413887, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4683754444122314, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06888013333082199, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.242233753204346, "metrics/particle/momentum/class_2/assigned/samples": 2438.0, "metrics/particle/momentum/class_2/efficiency": 0.5102452039718628, "metrics/particle/momentum/class_2/event/energy_bias": -11.661741256713867, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.613020896911621, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.37885046005249, "metrics/particle/momentum/class_2/event/px_bias": 0.8177577257156372, "metrics/particle/momentum/class_2/event/py_bias": -0.07800103724002838, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.983933448791504, "metrics/particle/momentum/class_2/event/pz_bias": -1.1443971395492554, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.240142822265625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02136216126382351, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15506254136562347, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15405137836933136, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.018296292051672935, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1944865882396698, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022536187898367643, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2805834412574768, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.005561416503041983, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0562748908996582, "metrics/particle/momentum/class_3/accepted/samples": 11537.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3069840967655182, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9929012060165405, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8913975954055786, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7340940237045288, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17051728069782257, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.255386352539062, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009541334584355354, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6162135601043701, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.045392315834760666, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5861315727233887, "metrics/particle/momentum/class_3/assigned/samples": 14673.0, "metrics/particle/momentum/class_3/efficiency": 0.7586637735366821, "metrics/particle/momentum/class_3/event/energy_bias": 3.873840093612671, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.001240730285645, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.481257438659668, "metrics/particle/momentum/class_3/event/px_bias": -0.4590298533439636, "metrics/particle/momentum/class_3/event/py_bias": 0.9708925485610962, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.132094383239746, "metrics/particle/momentum/class_3/event/pz_bias": -1.6846556663513184, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.200666427612305, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028227340430021286, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1568867415189743, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1529109925031662, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054222315549850464, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19622759521007538, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01075268816202879, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8766454458236694, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3331136405467987, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.612999439239502, "metrics/particle/momentum/class_4/accepted/samples": 279.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.34411171078681946, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.4412224292755127, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.709763288497925, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.6057236194610596, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16853933036327362, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.992152214050293, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019662922248244286, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1928882598876953, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.026940029114484787, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.8216183185577393, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.7664835453033447, "metrics/particle/momentum/class_4/event/energy_bias": -1.100274682044983, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8122148513793945, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9563322067260742, "metrics/particle/momentum/class_4/event/px_bias": 0.09842652082443237, "metrics/particle/momentum/class_4/event/py_bias": 0.17510366439819336, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7231957912445068, "metrics/particle/momentum/class_4/event/pz_bias": 0.4515645205974579, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.263981342315674, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018841376528143883, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1351180523633957, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13680435717105865, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.006621631793677807, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1766493171453476, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03921568766236305, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6901401281356812, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.7006155848503113, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3948514461517334, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03460647538304329, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15177787840366364, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1547219455242157, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.008556660264730453, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.21900111436843872, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0569620244204998, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9958767890930176, "metrics/particle/momentum/class_5/assigned/pz_bias": -1.018428087234497, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.21505880355835, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": -0.673485517501831, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9728636741638184, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.502927541732788, "metrics/particle/momentum/class_5/event/px_bias": -0.18065145611763, "metrics/particle/momentum/class_5/event/py_bias": 0.13501261174678802, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0412428379058838, "metrics/particle/momentum/class_5/event/pz_bias": -0.16140484809875488, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.443686008453369, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3688007593154907, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.36980491876602173, "metrics/set_slot/angular_recall_all_dr0p10": 0.9477352499961853, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8272308111190796, "metrics/set_slot/soft_count_bias_mean": 0.10282456874847412}, "step_train_time": 5.559423208236694, "step_valid_time": 74547.38262224197, "step_total_time": 74552.94204568863, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0089480825739576, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1343626046883981, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2864, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514370664023786, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9520661157024793, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9517514871116987, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047933884297520664, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461513049223654, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9467768595041323, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9464639788499669, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514370664023786, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007066106313127, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1349113181272682, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2124, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9412024756852343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950870924519875, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9460119973339258, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04912907548012506, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9389920424403183, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9486377847253238, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9437902688291491, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9412024756852343, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021183968210767, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12106906292885755, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.937111801242236, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9444444444444444, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9407638347622759, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05555555555555555, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9324534161490683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9397496087636933, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.936087295401403, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.937111801242236} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_25000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f478eaa686dc7e9a928ac7d9cfd22f6b65f8275 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1843850672730718, "train": {"Classification_binary": 0.15507446229457855, "Cardinality": 0.1385088413953781, "Classification": 0.07371991872787476, "Regression_pt": 0.11737877130508423, "Regression_eta": 0.001505599939264357, "Regression_sin_phi": 0.0008841576054692268, "Regression_cos_phi": 0.0008156225667335093, "Regression_energy": 0.11565259099006653, "HitGrouping": 0.15329882502555847, "Auxiliary": 0.1961725503206253, "Total": 0.9796370267868042, "Total_unweighted": 0.46503112342907116}, "train_group": {}, "valid": {"Classification_binary": 0.16859694321950278, "Cardinality": 0.12641085187594095, "Classification": 0.08011439939339955, "Regression_pt": 0.12940242886543274, "Regression_eta": 0.0017027476181586583, "Regression_sin_phi": 0.0009414136099318663, "Regression_cos_phi": 0.0009380466459939877, "Regression_energy": 0.12829520304997763, "HitGrouping": 0.1618192195892334, "Auxiliary": 0.20920618375142416, "Total": 1.0385364691416423, "Total_unweighted": 0.5099911824023972, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8403555154800415, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5653780102729797, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3269103765487671, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5693138241767883, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5599493384361267, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3500519394874573, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7902251482009888, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5168054103851318, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3358183801174164, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5239883661270142, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5211013555526733, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.42832937836647034, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.850422739982605, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5473776459693909, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2981097102165222, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5497968196868896, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5464887619018555, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33452123403549194, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8417835831642151, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5504277944564819, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.31075987219810486, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5537623763084412, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5485430955886841, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3479331433773041, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026667864993214607, "metrics/hit_grouping/calorimeter/background_recall": 0.6472446322441101, "metrics/hit_grouping/calorimeter/completeness": 0.6182445287704468, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6262985467910767, "metrics/hit_grouping/calorimeter/energy_purity": 0.6191006302833557, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.35887402296066284, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8555355668067932, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5628636479377747, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24801433086395264, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5693315863609314, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5615663528442383, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.30265751481056213, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8462614417076111, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5723713636398315, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3210819959640503, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.575531542301178, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5706241726875305, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3521348834037781, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.808009147644043, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5012379288673401, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4328150749206543, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.49812188744544983, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.49796658754348755, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4393881857395172, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8320154547691345, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5797458291053772, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.36250388622283936, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5855110883712769, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5900150537490845, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3892245292663574, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8114613890647888, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6564053297042847, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4693877696990967, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.649386465549469, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6543487906455994, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4039769768714905, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8561651706695557, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5053478479385376, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2348332703113556, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5073940753936768, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.490418940782547, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2963706851005554, "metrics/hit_grouping/calorimeter/purity": 0.6096885800361633, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8128693103790283, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.14523661136627197, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1261538416147232, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1415073275566101, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0976433977484703, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03384615480899811, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.801896870136261, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3153286278247833, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.13055872917175293, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.31475117802619934, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.305157870054245, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23147092759609222, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8446940183639526, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5725860595703125, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32505902647972107, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5762962698936462, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5721521973609924, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.36132606863975525, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7379969954490662, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41049012541770935, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5086899399757385, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5404735207557678, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.576060950756073, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7950748801231384, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3107122778892517, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5558980703353882, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5722053050994873, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4391475021839142, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6644195914268494, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4311254620552063, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4234544038772583, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.46917489171028137, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6808695197105408, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6986788511276245, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41446250677108765, "metrics/hit_grouping/tracker/all/object_completeness": 0.4615492522716522, "metrics/hit_grouping/tracker/all/object_purity": 0.5003875494003296, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6273851990699768, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00688409898430109, "metrics/hit_grouping/tracker/background_recall": 0.0520642064511776, "metrics/hit_grouping/tracker/completeness": 0.477560818195343, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4519721269607544, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.730597972869873, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4026423692703247, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5427901744842529, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5710620880126953, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5896127820014954, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6735022068023682, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4325147569179535, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4261423349380493, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.47756487131118774, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6624785661697388, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6545730829238892, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42152103781700134, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3188253343105316, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.36774876713752747, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6739482283592224, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7012820243835449, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4255931079387665, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.45766159892082214, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5042235255241394, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6170920133590698, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6845148205757141, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.40279626846313477, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4181744456291199, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5077955722808838, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6491345167160034, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6977783441543579, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4020135700702667, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4742406904697418, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4940876364707947, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6369702816009521, "metrics/hit_grouping/tracker/purity": 0.4561087191104889, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8902605772018433, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03735881671309471, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.10647544264793396, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11467718333005905, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6910582780838013, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2726801335811615, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.34046944975852966, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.34884724020957947, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5438050627708435, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6735138297080994, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4818487763404846, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5235847234725952, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5700543522834778, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7117643356323242, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5383866429328918, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.652424693107605, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7337806224822998, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6452780961990356, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05265931040048599, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1144733652472496, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.052738189697266, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2941824495792389, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16374260187149048, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8636130094528198, "metrics/particle/class_1/pid_accuracy": 0.9690471291542053, "metrics/particle/class_2/efficiency": 0.49966344237327576, "metrics/particle/class_2/pid_accuracy": 0.7132914066314697, "metrics/particle/class_3/efficiency": 0.7460579872131348, "metrics/particle/class_3/pid_accuracy": 0.9708994626998901, "metrics/particle/class_4/efficiency": 0.6900866031646729, "metrics/particle/class_4/pid_accuracy": 0.5230125784873962, "metrics/particle/class_5/efficiency": 0.906000018119812, "metrics/particle/class_5/pid_accuracy": 0.5695364475250244, "metrics/particle/count/bias_mean": -2.998046875, "metrics/particle/count/mae": 4.259765625, "metrics/particle/count/prediction_mean": 67.05794525146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06577830761671066, "metrics/particle/event/energy_response_mean": 0.9817979335784912, "metrics/particle/event/met_abs_error": 7.347422122955322, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053482379764318466, "metrics/particle/event/scalar_pt_response_mean": 0.9986433386802673, "metrics/particle/event/vector_pt_closure": 0.05871650204062462, "metrics/particle/matched/delta_eta_abs_mean": 0.016326431185007095, "metrics/particle/matched/delta_phi_abs_mean": 0.016204850748181343, "metrics/particle/matched/delta_r_mean": 0.025837771594524384, "metrics/particle/matched/energy_relative_abs_mean": 0.14448082447052002, "metrics/particle/matched/pid_accuracy": 0.9503015875816345, "metrics/particle/matched/pt_relative_abs_mean": 0.14658010005950928, "metrics/particle/matching/duplicate_fraction": 0.05819360911846161, "metrics/particle/matching/efficiency": 0.7810623645782471, "metrics/particle/matching/f1": 0.7981405854225159, "metrics/particle/matching/purity": 0.8159823417663574, "metrics/particle/matching/target_coverage_dr0p05": 0.7147742509841919, "metrics/particle/matching/target_coverage_dr0p10": 0.8318030834197998, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5832202434539795, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8197777271270752, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025350701063871384, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14293594658374786, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14738799631595612, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.04134267941117287, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1875329166650772, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006511187180876732, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6974461078643799, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08844173699617386, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8454647064208984, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8447.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23648406565189362, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9962687492370605, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1233230829238892, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9896596670150757, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13648833334445953, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.910884857177734, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014599255286157131, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0243030786514282, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12431655824184418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.847923755645752, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10206.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7724030613899231, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3240348100662231, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.159027099609375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9913713932037354, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.21867965161800385, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8726716637611389, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.9019551277160645, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.0533695220947266, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.717157363891602, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024530423805117607, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14652970433235168, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.152474045753479, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03986102342605591, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1932620108127594, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026007140055298805, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4796085357666016, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.26907697319984436, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7626657485961914, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1961.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3946121633052826, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2166907787322998, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.620072364807129, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4816653728485107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.16607071459293365, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 25.729467391967773, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05721096694469452, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3729381561279297, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.13315661251544952, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.184455871582031, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2517.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6491228342056274, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.219888210296631, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.886892318725586, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2598631381988525, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10256364941596985, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.13973373174667358, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.71645450592041, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.7212709188461304, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.382652282714844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025983817875385284, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14483891427516937, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14543795585632324, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.037655264139175415, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18598394095897675, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008143796003423631, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24544757604599, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.007028348743915558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7723913192749023, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17191.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21610461175441742, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3000175952911377, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9736525416374207, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8317461013793945, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13255970180034637, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.612476348876953, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005392912309616804, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4515937566757202, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03971761837601662, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8715816736221313, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20768.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8019312620162964, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.37943705916404724, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.998043060302734, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.627292633056641, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6605386734008789, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.4331858158111572, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.915397882461548, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.6115641593933105, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.378655433654785, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025686778128147125, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14437662065029144, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1465347409248352, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03894057124853134, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18698455393314362, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004347983747720718, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.47147834300994873, "metrics/particle/momentum/all/accepted/pz_bias": 0.05056530237197876, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5556094646453857, "metrics/particle/momentum/all/accepted/samples": 27599.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23573066294193268, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2011911869049072, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1429989337921143, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0038673877716064, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13627541065216064, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.210803985595703, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012092800810933113, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7705181837081909, "metrics/particle/momentum/all/assigned/pz_bias": 0.07252053916454315, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.741938591003418, "metrics/particle/momentum/all/assigned/samples": 33491.0, "metrics/particle/momentum/all/efficiency": 0.7797649502754211, "metrics/particle/momentum/all/event/energy_bias": -5.164485931396484, "metrics/particle/momentum/all/event/p3_error_mean": 18.885515213012695, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.644508361816406, "metrics/particle/momentum/all/event/px_bias": 0.7766547203063965, "metrics/particle/momentum/all/event/py_bias": 3.166123628616333, "metrics/particle/momentum/all/event/pz_abs_mean": 13.855209350585938, "metrics/particle/momentum/all/event/pz_bias": 4.386205196380615, "metrics/particle/momentum/all/event/pz_residual_rms": 21.012348175048828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02815336547791958, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13182978332042694, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13547779619693756, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.051224235445261, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17421944439411163, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0038146760780364275, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.48720261454582214, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06496875733137131, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4955253601074219, "metrics/particle/momentum/class_1/accepted/samples": 14418.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13397088646888733, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3877190947532654, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4080803692340851, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.29756826162338257, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07722198218107224, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.636504888534546, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007463146932423115, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6332770586013794, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.09718193113803864, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0877466201782227, "metrics/particle/momentum/class_1/assigned/samples": 16213.0, "metrics/particle/momentum/class_1/efficiency": 0.864025890827179, "metrics/particle/momentum/class_1/event/energy_bias": 5.773056983947754, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.853858947753906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.51805591583252, "metrics/particle/momentum/class_1/event/px_bias": 1.5106230974197388, "metrics/particle/momentum/class_1/event/py_bias": 2.9117980003356934, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.914583206176758, "metrics/particle/momentum/class_1/event/pz_bias": 3.742259979248047, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.21917724609375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.030652442947030067, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18441574275493622, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18568472564220428, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.008934570476412773, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22588449716567993, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021145975217223167, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5060787200927734, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.020130399614572525, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5922904014587402, "metrics/particle/momentum/class_2/accepted/samples": 1466.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5363495349884033, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7826225757598877, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0029776096343994, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7000415921211243, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28735631704330444, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.476052284240723, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05023414269089699, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4261350631713867, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13464364409446716, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.146791934967041, "metrics/particle/momentum/class_2/assigned/samples": 2349.0, "metrics/particle/momentum/class_2/efficiency": 0.4924420416355133, "metrics/particle/momentum/class_2/event/energy_bias": -10.934755325317383, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.675068855285645, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.742655277252197, "metrics/particle/momentum/class_2/event/px_bias": -0.029107170179486275, "metrics/particle/momentum/class_2/event/py_bias": -0.9266349673271179, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.753060340881348, "metrics/particle/momentum/class_2/event/pz_bias": 0.7948683500289917, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.36763572692871, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02194058895111084, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15485742688179016, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15527519583702087, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027700256556272507, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19646809995174408, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0024734982289373875, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29388198256492615, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02982194535434246, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0776149034500122, "metrics/particle/momentum/class_3/accepted/samples": 11320.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30026116967201233, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.144298791885376, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9636980295181274, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8189765214920044, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1766703575849533, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.33561897277832, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010535070672631264, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6330040693283081, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.038536373525857925, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4477264881134033, "metrics/particle/momentum/class_3/assigned/samples": 14428.0, "metrics/particle/momentum/class_3/efficiency": 0.7443940043449402, "metrics/particle/momentum/class_3/event/energy_bias": 1.9975298643112183, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.714067459106445, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.53277587890625, "metrics/particle/momentum/class_3/event/px_bias": -0.5874786972999573, "metrics/particle/momentum/class_3/event/py_bias": 0.8479048609733582, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.671988487243652, "metrics/particle/momentum/class_3/event/pz_bias": -0.3626871705055237, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.655485153198242, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02612716518342495, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15902189910411835, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15933968126773834, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.045840516686439514, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20669430494308472, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01568627543747425, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9596502184867859, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4250757396221161, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.3053011894226074, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3585950434207916, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.2777724266052246, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.726276159286499, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.605842113494873, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21098266541957855, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.54713249206543, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0202312134206295, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.12395179271698, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.20240768790245056, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7926058769226074, "metrics/particle/momentum/class_4/assigned/samples": 346.0, "metrics/particle/momentum/class_4/efficiency": 0.7005494236946106, "metrics/particle/momentum/class_4/event/energy_bias": -0.9301661252975464, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.787036180496216, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9141039848327637, "metrics/particle/momentum/class_4/event/px_bias": -0.06356357038021088, "metrics/particle/momentum/class_4/event/py_bias": 0.08375509828329086, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7237484455108643, "metrics/particle/momentum/class_4/event/pz_bias": 0.42257434129714966, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.001211166381836, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021769586950540543, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14313346147537231, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14523115754127502, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.015603930689394474, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18293125927448273, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.014285714365541935, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4891083240509033, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.118983693420887, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.8303346633911133, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.042963989078998566, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.21014365553855896, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20933419466018677, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.037649352103471756, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09677419066429138, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3586758077144623, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04516129195690155, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.046719789505005, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5750894546508789, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.565962314605713, "metrics/particle/momentum/class_5/assigned/samples": 155.0, "metrics/particle/momentum/class_5/efficiency": 0.8805031180381775, "metrics/particle/momentum/class_5/event/energy_bias": -1.070150375366211, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.138031005859375, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6609503030776978, "metrics/particle/momentum/class_5/event/px_bias": -0.053818944841623306, "metrics/particle/momentum/class_5/event/py_bias": 0.24930092692375183, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0812278985977173, "metrics/particle/momentum/class_5/event/pz_bias": -0.21081097424030304, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7002930641174316, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3807010352611542, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.38323211669921875, "metrics/set_slot/angular_recall_all_dr0p10": 0.9478188753128052, "metrics/set_slot/angular_recall_selected_dr0p10": 0.819489598274231, "metrics/set_slot/soft_count_bias_mean": -1.2038384675979614}, "step_train_time": 5.641350030899048, "step_valid_time": 77754.37027573586, "step_total_time": 77760.01162624359, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987873852378312, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1460675425415039, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3015, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2858, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2829, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9441691443673604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.947927031509121, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9460443561734525, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05207296849087894, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9345887016848364, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9383084577114428, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9364448857994042, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9441691443673604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010158250756889, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15433662062981157, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2256, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2143, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9473916887709991, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9499113475177305, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9486498450641877, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050088652482269506, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9438549955791335, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9463652482269503, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9451084550686144, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9473916887709991, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960109243161732, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15340123592086896, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1180, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9161490683229814, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9240407204385278, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9200779727095516, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0759592795614722, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9114906832298136, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9193422083007048, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9153996101364522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9161490683229814} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_26000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..6567ffe2d96bfcd5720eda9581d93b14cd839c7d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2317604699639948, "train": {"Classification_binary": 0.14284682273864746, "Cardinality": 0.09833166003227234, "Classification": 0.07799386978149414, "Regression_pt": 0.12692895531654358, "Regression_eta": 0.0016363277100026608, "Regression_sin_phi": 0.0008878231164999306, "Regression_cos_phi": 0.000937685661483556, "Regression_energy": 0.12562671303749084, "HitGrouping": 0.15396025776863098, "Auxiliary": 0.1937975138425827, "Total": 0.9516079425811768, "Total_unweighted": 0.4768581973621622}, "train_group": {}, "valid": {"Classification_binary": 0.16359190146128336, "Cardinality": 0.11333200335502625, "Classification": 0.07832565903663635, "Regression_pt": 0.11734512448310852, "Regression_eta": 0.0015669218264520168, "Regression_sin_phi": 0.0008596327000608047, "Regression_cos_phi": 0.0008669892946879069, "Regression_energy": 0.1170094112555186, "HitGrouping": 0.1555980642636617, "Auxiliary": 0.19928898413976034, "Total": 0.9756258328755697, "Total_unweighted": 0.47956564005774754, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8408821821212769, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5794045925140381, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.30528005957603455, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.586054801940918, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5858719348907471, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34797394275665283, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7907216548919678, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5332579612731934, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32395797967910767, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5410605669021606, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5500985383987427, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4303625822067261, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8506982326507568, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5550581812858582, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2867472469806671, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5588445663452148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5552534461021423, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32460489869117737, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8421597480773926, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5609372854232788, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.296048641204834, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5660054087638855, "metrics/hit_grouping/calorimeter/all/object_purity": 0.564643919467926, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34161093831062317, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02686372399330139, "metrics/hit_grouping/calorimeter/background_recall": 0.6495456695556641, "metrics/hit_grouping/calorimeter/completeness": 0.6379674673080444, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6483700275421143, "metrics/hit_grouping/calorimeter/energy_purity": 0.64106285572052, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.34293416142463684, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8576788306236267, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5719651579856873, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23630934953689575, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5798642635345459, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.571592390537262, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2915497124195099, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8473997116088867, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5798904299736023, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.30664727091789246, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5847005844116211, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5864397883415222, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.344856858253479, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8038505911827087, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5178436040878296, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.41145241260528564, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5171889662742615, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5272230505943298, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4441916346549988, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8316531777381897, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6001340746879578, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.34992215037345886, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6081884503364563, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6213319301605225, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3851136863231659, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8201862573623657, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6831303834915161, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.45002615451812744, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6789634823799133, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6817203164100647, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3799058198928833, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8562668561935425, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5031619071960449, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21842774748802185, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5062015652656555, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48871928453445435, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2899424135684967, "metrics/hit_grouping/calorimeter/purity": 0.6292651891708374, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8126989603042603, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11799468100070953, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11384615302085876, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11857447773218155, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08863291889429092, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.035384614020586014, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7958405613899231, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.28426072001457214, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11630558967590332, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2850542962551117, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2706027030944824, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22177879512310028, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.845451831817627, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.586291491985321, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.31027019023895264, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5917012095451355, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5917017459869385, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3550301790237427, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7411441802978516, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41049012541770935, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5272746682167053, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.568025529384613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5730723142623901, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7998837232589722, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3438025712966919, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5765784978866577, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5725545883178711, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4167134165763855, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6825577616691589, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.41316846013069153, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44042980670928955, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.48699507117271423, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6570843458175659, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7109562754631042, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40648025274276733, "metrics/hit_grouping/tracker/all/object_completeness": 0.47934532165527344, "metrics/hit_grouping/tracker/all/object_purity": 0.5198124051094055, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6102871894836426, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.003114162478595972, "metrics/hit_grouping/tracker/background_recall": 0.026002487167716026, "metrics/hit_grouping/tracker/completeness": 0.4937743842601776, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.439590185880661, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7475101351737976, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39835989475250244, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5581257343292236, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5743631720542908, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5654670000076294, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.686837911605835, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41633352637290955, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.44690969586372375, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5078433752059937, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6426803469657898, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6554372310638428, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41403722763061523, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33892640471458435, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4114342927932739, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.675364077091217, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7136460542678833, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4047447741031647, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4778050482273102, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5372865796089172, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5998382568359375, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6887802481651306, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4221038520336151, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45234498381614685, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5389161109924316, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6371504664421082, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7113513946533203, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40599390864372253, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4860994815826416, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.49368882179260254, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6191758513450623, "metrics/hit_grouping/tracker/purity": 0.4698105454444885, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.889567494392395, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.056038226932287216, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1370597928762436, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13494113087654114, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08210252225399017, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6969887018203735, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27475374937057495, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3618375360965729, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3746456801891327, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5406946539878845, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6882762908935547, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46909862756729126, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.539236307144165, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.588639497756958, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6899828910827637, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5751850008964539, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6870650053024292, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7763696312904358, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6397137641906738, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.08945765346288681, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1089089959859848, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.00469207763672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3194076418876648, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18896780908107758, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8913929462432861, "metrics/particle/class_1/pid_accuracy": 0.9689685106277466, "metrics/particle/class_2/efficiency": 0.541283369064331, "metrics/particle/class_2/pid_accuracy": 0.770569920539856, "metrics/particle/class_3/efficiency": 0.7634565234184265, "metrics/particle/class_3/pid_accuracy": 0.9685559272766113, "metrics/particle/class_4/efficiency": 0.7305101156234741, "metrics/particle/class_4/pid_accuracy": 0.52173912525177, "metrics/particle/class_5/efficiency": 0.9340000152587891, "metrics/particle/class_5/pid_accuracy": 0.6231263279914856, "metrics/particle/count/bias_mean": -1.0852864980697632, "metrics/particle/count/mae": 3.2545573711395264, "metrics/particle/count/prediction_mean": 68.970703125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.059630777686834335, "metrics/particle/event/energy_response_mean": 0.9775448441505432, "metrics/particle/event/met_abs_error": 6.029914379119873, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04718971624970436, "metrics/particle/event/scalar_pt_response_mean": 0.9945993423461914, "metrics/particle/event/vector_pt_closure": 0.05069996044039726, "metrics/particle/matched/delta_eta_abs_mean": 0.01489072386175394, "metrics/particle/matched/delta_phi_abs_mean": 0.01534198597073555, "metrics/particle/matched/delta_r_mean": 0.02403734251856804, "metrics/particle/matched/energy_relative_abs_mean": 0.13665294647216797, "metrics/particle/matched/pid_accuracy": 0.9519777297973633, "metrics/particle/matched/pt_relative_abs_mean": 0.13757066428661346, "metrics/particle/matching/duplicate_fraction": 0.05695730447769165, "metrics/particle/matching/efficiency": 0.8056149482727051, "metrics/particle/matching/f1": 0.8119038343429565, "metrics/particle/matching/purity": 0.8182916641235352, "metrics/particle/matching/target_coverage_dr0p05": 0.7442429065704346, "metrics/particle/matching/target_coverage_dr0p10": 0.8517740368843079, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6262568831443787, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8439213633537292, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023359838873147964, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13456088304519653, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13518258929252625, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.026594677940011024, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17617854475975037, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007741347886621952, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6721127033233643, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.025064099580049515, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.887040376663208, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8784.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24682076275348663, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0598363876342773, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0312660932540894, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8912582397460938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1266147345304489, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.174725532531738, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014152735471725464, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9736299514770508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05114240571856499, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.927966356277466, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10528.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8032187223434448, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3345035314559937, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.8770599365234375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.465273141860962, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.009169749915599823, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.23329687118530273, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.788586139678955, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1124283075332642, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.045239448547363, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022044872865080833, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13897521793842316, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13654178380966187, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02166166342794895, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1769109070301056, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026239067316055298, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3571604490280151, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10781662911176682, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.644785165786743, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2058.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4618038237094879, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1364812850952148, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.343428134918213, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1824984550476074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17719167470932007, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.73923110961914, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05943536385893822, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.411008358001709, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2156747579574585, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.595004081726074, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2692.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6812313795089722, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.7440285682678223, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.940199851989746, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2366209030151367, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.022463493049144745, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08044242858886719, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.781611919403076, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.709119975566864, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.357450485229492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02411467768251896, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13716396689414978, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13815322518348694, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025576051324605942, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1785806119441986, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010761214653030038, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2383318543434143, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008149531669914722, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7713169455528259, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17656.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2091795653104782, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1182687282562256, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8990058898925781, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7549684643745422, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12716488540172577, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.898519515991211, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004128113854676485, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.41880273818969727, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02971862442791462, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.859108805656433, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21075.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8236227035522461, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.1411012411117554, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.995163917541504, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.788989067077637, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2780689001083374, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.519840955734253, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.493185043334961, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5044042468070984, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.750180244445801, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023732539266347885, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13649243116378784, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13712123036384583, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02560734562575817, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17772312462329865, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004947715438902378, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4528339207172394, "metrics/particle/momentum/all/accepted/pz_bias": 0.02056066319346428, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5574098825454712, "metrics/particle/momentum/all/accepted/samples": 28498.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24056462943553925, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1017606258392334, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.052988052368164, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9088617563247681, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13092286884784698, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.703213691711426, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011546872556209564, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7455045580863953, "metrics/particle/momentum/all/assigned/pz_bias": 0.05089207738637924, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.858288049697876, "metrics/particle/momentum/all/assigned/samples": 34295.0, "metrics/particle/momentum/all/efficiency": 0.8051646947860718, "metrics/particle/momentum/all/event/energy_bias": -6.219634056091309, "metrics/particle/momentum/all/event/p3_error_mean": 16.979055404663086, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.455171585083008, "metrics/particle/momentum/all/event/px_bias": -0.3097020983695984, "metrics/particle/momentum/all/event/py_bias": -1.8335803747177124, "metrics/particle/momentum/all/event/pz_abs_mean": 12.647502899169922, "metrics/particle/momentum/all/event/pz_bias": 2.3259525299072266, "metrics/particle/momentum/all/event/pz_residual_rms": 19.55662727355957, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02527560107409954, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11774531751871109, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11901095509529114, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.025453662499785423, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15647664666175842, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004022256471216679, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44827941060066223, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0324309766292572, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4030758142471313, "metrics/particle/momentum/class_1/accepted/samples": 14917.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12730905413627625, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.28750982880592346, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.33280444145202637, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21333637833595276, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06235172599554062, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9177050590515137, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007482206914573908, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5872008800506592, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.05492601543664932, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9787578582763672, "metrics/particle/momentum/class_1/assigned/samples": 16439.0, "metrics/particle/momentum/class_1/efficiency": 0.8939294219017029, "metrics/particle/momentum/class_1/event/energy_bias": 4.299706935882568, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.756128311157227, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.98408031463623, "metrics/particle/momentum/class_1/event/px_bias": 0.7982313632965088, "metrics/particle/momentum/class_1/event/py_bias": -1.2107030153274536, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.917293548583984, "metrics/particle/momentum/class_1/event/pz_bias": 1.938293218612671, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.30706024169922, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029041800647974014, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18810470402240753, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18954546749591827, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026188256219029427, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2303709238767624, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02338811568915844, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5415210723876953, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0855545774102211, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8686375617980957, "metrics/particle/momentum/class_2/accepted/samples": 1582.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6492440700531006, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6843740940093994, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8961580991744995, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5409099459648132, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2857702970504761, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.814826250076294, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.048608388751745224, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4864978790283203, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09765183180570602, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.471850395202637, "metrics/particle/momentum/class_2/assigned/samples": 2551.0, "metrics/particle/momentum/class_2/efficiency": 0.5314074754714966, "metrics/particle/momentum/class_2/event/energy_bias": -10.361464500427246, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.838583946228027, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.155345916748047, "metrics/particle/momentum/class_2/event/px_bias": -0.17516258358955383, "metrics/particle/momentum/class_2/event/py_bias": 0.5346134305000305, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.232778549194336, "metrics/particle/momentum/class_2/event/pz_bias": 0.5212775468826294, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.359344482421875, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020999908447265625, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15323543548583984, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15276144444942474, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03247664123773575, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1938507854938507, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002935843076556921, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2833330035209656, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01763995550572872, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0592167377471924, "metrics/particle/momentum/class_3/accepted/samples": 11581.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2960980236530304, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.058220386505127, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8661898374557495, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7303186655044556, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1797957867383957, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.864479064941406, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009128406643867493, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6000895500183105, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03157789260149002, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.635667562484741, "metrics/particle/momentum/class_3/assigned/samples": 14789.0, "metrics/particle/momentum/class_3/efficiency": 0.7615571618080139, "metrics/particle/momentum/class_3/event/energy_bias": 0.7862085103988647, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.63296890258789, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.064905643463135, "metrics/particle/momentum/class_3/event/px_bias": -0.6663543581962585, "metrics/particle/momentum/class_3/event/py_bias": -1.0263563394546509, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.05240535736084, "metrics/particle/momentum/class_3/event/pz_bias": -0.6279712319374084, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.811391830444336, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027258846908807755, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14599332213401794, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15012334287166595, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04874729737639427, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19381394982337952, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.014814814552664757, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7489215135574341, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2923131287097931, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8301444053649902, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3141172528266907, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.364516019821167, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.033830404281616, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9218437671661377, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1955307275056839, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.092010498046875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01955307275056839, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.034122347831726, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.41328468918800354, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.24137544631958, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.7417582273483276, "metrics/particle/momentum/class_4/event/energy_bias": -0.621895432472229, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8571043014526367, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.035977363586426, "metrics/particle/momentum/class_4/event/px_bias": 0.1427246779203415, "metrics/particle/momentum/class_4/event/py_bias": -0.13489574193954468, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6368745565414429, "metrics/particle/momentum/class_4/event/pz_bias": 0.45354312658309937, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.869601249694824, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01885002665221691, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14685742557048798, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15452389419078827, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.015012924559414387, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1943385899066925, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04054053872823715, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9979923963546753, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.30878815054893494, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.8400356769561768, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06114543229341507, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17183727025985718, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.17699280381202698, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.030458208173513412, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04430379718542099, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2380894124507904, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04430379718542099, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.0638413429260254, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.1370643973350525, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.955507278442383, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9308176040649414, "metrics/particle/momentum/class_5/event/energy_bias": -0.3221893906593323, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0481579303741455, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.569826602935791, "metrics/particle/momentum/class_5/event/px_bias": -0.40914127230644226, "metrics/particle/momentum/class_5/event/py_bias": 0.003761296160519123, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0681620836257935, "metrics/particle/momentum/class_5/event/pz_bias": 0.0408100001513958, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.36553692817688, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3606877028942108, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3623325526714325, "metrics/set_slot/angular_recall_all_dr0p10": 0.9513781666755676, "metrics/set_slot/angular_recall_selected_dr0p10": 0.835975706577301, "metrics/set_slot/soft_count_bias_mean": 0.8425040245056152}, "step_train_time": 5.405218839645386, "step_valid_time": 80789.28672504425, "step_total_time": 80794.69194436073, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032255813993392, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13197298041396954, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2903, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2877, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9590353485298976, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9533661740558292, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561923583662714, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04663382594417077, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504459861248761, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9448275862068966, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9476284584980237, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9590353485298976, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9941900485688124, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12944415737879766, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2146, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2141, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9487179487179487, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9495575221238938, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9491375497567448, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050442477876106194, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9465075154730327, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9473451327433628, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9469261388766033, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9487179487179487, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9907615722049492, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12716419774130927, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1196, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1191, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9285714285714286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9292929292929293, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9289320388349515, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0707070707070707, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9246894409937888, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9254079254079254, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9250485436893204, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9285714285714286} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_27000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..f85d2c115d342913816ed931b0657ec4a6a030d9 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2791358726549176, "train": {"Classification_binary": 0.14375315606594086, "Cardinality": 0.1204231008887291, "Classification": 0.08711493015289307, "Regression_pt": 0.11961551010608673, "Regression_eta": 0.001410515746101737, "Regression_sin_phi": 0.000898207479622215, "Regression_cos_phi": 0.0008568224729970098, "Regression_energy": 0.11812760680913925, "HitGrouping": 0.1591932475566864, "Auxiliary": 0.19820362329483032, "Total": 0.9700059294700623, "Total_unweighted": 0.47177674883278087}, "train_group": {}, "valid": {"Classification_binary": 0.15541502833366394, "Cardinality": 0.10922221342722575, "Classification": 0.07828847567240398, "Regression_pt": 0.11677129069964091, "Regression_eta": 0.001533674697081248, "Regression_sin_phi": 0.0008622996198634306, "Regression_cos_phi": 0.0008797082118690014, "Regression_energy": 0.11590711275736491, "HitGrouping": 0.15436607599258423, "Auxiliary": 0.19489330053329468, "Total": 0.9570173422495524, "Total_unweighted": 0.46965758999188745, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8421850204467773, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5815198421478271, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2949844002723694, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5868134498596191, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5954251885414124, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3509020507335663, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7897690534591675, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.537398099899292, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32734665274620056, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5458383560180664, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.554872453212738, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4327346682548523, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.853052020072937, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.562461256980896, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.28519782423973083, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5661807656288147, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5684286952018738, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33126741647720337, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8438788056373596, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5663461089134216, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.29212766885757446, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5709956884384155, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5758999586105347, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3466869294643402, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026432139798998833, "metrics/hit_grouping/calorimeter/background_recall": 0.6461774110794067, "metrics/hit_grouping/calorimeter/completeness": 0.6383659839630127, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6473585367202759, "metrics/hit_grouping/calorimeter/energy_purity": 0.6399518251419067, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3342203199863434, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8620412945747375, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5741536617279053, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22747088968753815, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5824683308601379, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5828123092651367, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.29662585258483887, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.847256064414978, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5916527509689331, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.30458515882492065, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5952467322349548, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5984275937080383, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.35201358795166016, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.80191570520401, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5234481692314148, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4160030484199524, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5214395523071289, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5377929210662842, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.447098970413208, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8311574459075928, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6052902936935425, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.34761756658554077, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6121103167533875, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6282869577407837, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3915913999080658, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8231858015060425, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6683011054992676, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4385138750076294, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6664279103279114, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6799744367599487, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3731030821800232, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8602031469345093, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5114322900772095, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21374045312404633, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5145829916000366, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.506263017654419, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29503145813941956, "metrics/hit_grouping/calorimeter/purity": 0.6293792128562927, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8156000971794128, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.14492590725421906, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11230769008398056, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.14500366151332855, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.10749002546072006, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02769230678677559, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8200728297233582, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29965558648109436, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11801595985889435, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2992863655090332, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2908017337322235, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24515393376350403, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8458508253097534, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5906673073768616, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.305974543094635, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5957028865814209, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.602281391620636, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3593258261680603, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7346943616867065, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40092647075653076, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.519464373588562, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5706372857093811, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5808427929878235, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7862175703048706, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3123948276042938, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5622875094413757, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.590847373008728, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44924286007881165, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6693159341812134, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4266362190246582, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44045257568359375, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.494833767414093, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6752776503562927, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6998254060745239, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4088890850543976, "metrics/hit_grouping/tracker/all/object_completeness": 0.47568681836128235, "metrics/hit_grouping/tracker/all/object_purity": 0.5268793702125549, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6263933777809143, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005450547207146883, "metrics/hit_grouping/tracker/background_recall": 0.04181721433997154, "metrics/hit_grouping/tracker/completeness": 0.48847171664237976, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4356194734573364, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.729816198348999, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3938041031360626, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5493326783180237, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5919113755226135, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5895216464996338, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6780157685279846, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42680373787879944, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4456779956817627, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5090615749359131, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6622881889343262, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.656423032283783, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42334142327308655, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3440874516963959, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.40144872665405273, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6701051592826843, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6993585228919983, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42280733585357666, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4749704897403717, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5377730131149292, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6203271150588989, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7023049592971802, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4081225097179413, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4505073130130768, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5218974351882935, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.621171772480011, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6999977827072144, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39089205861091614, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.48104751110076904, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5135637521743774, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6352142095565796, "metrics/hit_grouping/tracker/purity": 0.46585702896118164, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8922048211097717, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05430060997605324, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.14578461647033691, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.14566950500011444, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6974301338195801, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2742353677749634, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3747609853744507, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.39378032088279724, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5386210680007935, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6739569902420044, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47240421175956726, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5320070385932922, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5938349366188049, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7111150622367859, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5824593305587769, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6990579962730408, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7922984957695007, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6347869634628296, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09673195332288742, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.10398218035697937, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.017200469970703, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2916637063026428, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1612238883972168, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8919865489006042, "metrics/particle/class_1/pid_accuracy": 0.9707637429237366, "metrics/particle/class_2/efficiency": 0.5192955136299133, "metrics/particle/class_2/pid_accuracy": 0.7494059205055237, "metrics/particle/class_3/efficiency": 0.782206654548645, "metrics/particle/class_3/pid_accuracy": 0.9744384288787842, "metrics/particle/class_4/efficiency": 0.7295476198196411, "metrics/particle/class_4/pid_accuracy": 0.5263852477073669, "metrics/particle/class_5/efficiency": 0.9399999976158142, "metrics/particle/class_5/pid_accuracy": 0.651063859462738, "metrics/particle/count/bias_mean": -1.3522135019302368, "metrics/particle/count/mae": 3.4381511211395264, "metrics/particle/count/prediction_mean": 68.70377349853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06662078946828842, "metrics/particle/event/energy_response_mean": 0.9845762252807617, "metrics/particle/event/met_abs_error": 6.859800815582275, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051418621093034744, "metrics/particle/event/scalar_pt_response_mean": 0.9990519881248474, "metrics/particle/event/vector_pt_closure": 0.054331351071596146, "metrics/particle/matched/delta_eta_abs_mean": 0.014714624732732773, "metrics/particle/matched/delta_phi_abs_mean": 0.01550726406276226, "metrics/particle/matched/delta_r_mean": 0.024021586403250694, "metrics/particle/matched/energy_relative_abs_mean": 0.13259629905223846, "metrics/particle/matched/pid_accuracy": 0.9549995064735413, "metrics/particle/matched/pt_relative_abs_mean": 0.13288229703903198, "metrics/particle/matching/duplicate_fraction": 0.05413677915930748, "metrics/particle/matching/efficiency": 0.8122130632400513, "metrics/particle/matching/f1": 0.8201280832290649, "metrics/particle/matching/purity": 0.8281989097595215, "metrics/particle/matching/target_coverage_dr0p05": 0.7504693269729614, "metrics/particle/matching/target_coverage_dr0p10": 0.8558444976806641, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6433470249176025, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8491626977920532, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02355612814426422, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1307128220796585, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13195252418518066, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011769212782382965, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17118658125400543, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005633802618831396, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6419200897216797, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04171295091509819, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7364636659622192, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8875.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23610299825668335, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8549469709396362, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9874300360679626, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8345740437507629, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11964675784111023, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.999465942382812, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012629379518330097, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9474014639854431, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03756782039999962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7992477416992188, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10531.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.811539888381958, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.9758569002151489, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.14237117767334, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6336400508880615, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.7530430555343628, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.24324899911880493, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.986616134643555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.6748549342155457, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.61563777923584, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023536667227745056, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13958889245986938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13970060646533966, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02591303363442421, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17937837541103363, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023680316284298897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2987287044525146, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08068405836820602, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.306612491607666, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2027.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.408271461725235, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0229343175888062, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.0957212448120117, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9398219585418701, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17298124730587006, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 22.06536865234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.058553386479616165, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.477069616317749, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.05404500663280487, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.816977024078369, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2613.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6709699034690857, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.136157512664795, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.157551765441895, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2889114618301392, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.041015900671482086, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1475241780281067, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.008288383483887, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1638928651809692, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.26978302001953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024031855165958405, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13172529637813568, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1317240446805954, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02055797353386879, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1717110276222229, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005042016855441034, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2184268683195114, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004057038109749556, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6607939600944519, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17850.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1879095882177353, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0388941764831543, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8343697190284729, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6866565346717834, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11536446958780289, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.40457534790039, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0035233062226325274, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3787306249141693, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002832339843735099, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5559455156326294, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21003.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8326724767684937, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.6618190407752991, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.652559280395508, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.384783744812012, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7659945487976074, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.133000373840332, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.7404391765594482, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.26700055599212646, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.414912223815918, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023850100114941597, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13196714222431183, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1323569118976593, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018222639337182045, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17210157215595245, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003721480257809162, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4253089725971222, "metrics/particle/momentum/all/accepted/pz_bias": 0.02108260616660118, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4045101404190063, "metrics/particle/momentum/all/accepted/samples": 28752.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21963506937026978, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9809431433677673, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9780951738357544, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8281695246696472, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12109409272670746, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.143561363220215, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010542653501033783, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7146790027618408, "metrics/particle/momentum/all/assigned/pz_bias": 0.01746373064815998, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.929427146911621, "metrics/particle/momentum/all/assigned/samples": 34147.0, "metrics/particle/momentum/all/efficiency": 0.8123410940170288, "metrics/particle/momentum/all/event/energy_bias": -4.498481750488281, "metrics/particle/momentum/all/event/p3_error_mean": 19.604761123657227, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.246716499328613, "metrics/particle/momentum/all/event/px_bias": 1.4780218601226807, "metrics/particle/momentum/all/event/py_bias": 1.5237736701965332, "metrics/particle/momentum/all/event/pz_abs_mean": 14.80247974395752, "metrics/particle/momentum/all/event/pz_bias": -0.75603848695755, "metrics/particle/momentum/all/event/pz_residual_rms": 25.73744773864746, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02616194635629654, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11056510359048843, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11157670617103577, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008253082633018494, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1471487581729889, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033581838943064213, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4388412833213806, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.010672403499484062, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3572396039962769, "metrics/particle/momentum/class_1/accepted/samples": 14889.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12750056385993958, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30211010575294495, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3056469261646271, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17112059891223907, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06253047287464142, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.7377166748046875, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00761823495849967, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6121315956115723, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.025357825681567192, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6044392585754395, "metrics/particle/momentum/class_1/assigned/samples": 16408.0, "metrics/particle/momentum/class_1/efficiency": 0.8922514319419861, "metrics/particle/momentum/class_1/event/energy_bias": 3.2248992919921875, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.459768295288086, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.567987442016602, "metrics/particle/momentum/class_1/event/px_bias": 1.7076448202133179, "metrics/particle/momentum/class_1/event/py_bias": 2.0767099857330322, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.30555248260498, "metrics/particle/momentum/class_1/event/pz_bias": 0.6996313333511353, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.926164627075195, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028680363669991493, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1817546784877777, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18291178345680237, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.01200035773217678, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22342874109745026, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022207707166671753, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3899166584014893, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08294586837291718, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.305676221847534, "metrics/particle/momentum/class_2/accepted/samples": 1531.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.609316349029541, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6981658935546875, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8630181550979614, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5414318442344666, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2809543311595917, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.514113664627075, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0526532307267189, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.432485580444336, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05051214620471001, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.950005531311035, "metrics/particle/momentum/class_2/assigned/samples": 2431.0, "metrics/particle/momentum/class_2/efficiency": 0.5142760872840881, "metrics/particle/momentum/class_2/event/energy_bias": -10.40361499786377, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.124515533447266, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.0135273933410645, "metrics/particle/momentum/class_2/event/px_bias": 0.782017707824707, "metrics/particle/momentum/class_2/event/py_bias": -0.4289435148239136, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.6143159866333, "metrics/particle/momentum/class_2/event/pz_bias": -1.2372970581054688, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.978553771972656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020320327952504158, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15187236666679382, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1516411453485489, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030782531946897507, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1920291781425476, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015113350236788392, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26238197088241577, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01677088439464569, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9251834154129028, "metrics/particle/momentum/class_3/accepted/samples": 11910.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25646576285362244, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7619233131408691, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7382597923278809, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5983537435531616, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1591646373271942, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.684165954589844, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006555825937539339, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5247684121131897, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.004248184617608786, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4421074390411377, "metrics/particle/momentum/class_3/assigned/samples": 14796.0, "metrics/particle/momentum/class_3/efficiency": 0.783191978931427, "metrics/particle/momentum/class_3/event/energy_bias": 3.474496603012085, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.774638175964355, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.098676681518555, "metrics/particle/momentum/class_3/event/px_bias": -0.7921181917190552, "metrics/particle/momentum/class_3/event/py_bias": -0.5293776392936707, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.100597381591797, "metrics/particle/momentum/class_3/event/pz_bias": -0.5707398653030396, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.818328857421875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026889242231845856, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14412014186382294, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13139159977436066, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04904228076338768, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17497779428958893, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007407407276332378, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7146947383880615, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.28045955300331116, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7508476972579956, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3404231667518616, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.006525754928589, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4680495262145996, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3664417266845703, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1859154999256134, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.967328071594238, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0797597169876099, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.20437398552894592, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.639645576477051, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.7417582273483276, "metrics/particle/momentum/class_4/event/energy_bias": -0.4009326696395874, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.605180025100708, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6812041997909546, "metrics/particle/momentum/class_4/event/px_bias": -0.04930182546377182, "metrics/particle/momentum/class_4/event/py_bias": -0.056519560515880585, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.704339623451233, "metrics/particle/momentum/class_4/event/pz_bias": 0.2861779034137726, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.244813442230225, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01992148719727993, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14563661813735962, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14934630692005157, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.018574189394712448, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18739299476146698, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019736841320991516, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6360315084457397, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2948046624660492, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.181337356567383, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07059641927480698, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3839028477668762, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.28988927602767944, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.135067880153656, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03184713423252106, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.615945816040039, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03184713423252106, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9052863121032715, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.30427438020706177, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.169003963470459, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": -0.39333000779151917, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.09844708442688, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5821107625961304, "metrics/particle/momentum/class_5/event/px_bias": -0.17022082209587097, "metrics/particle/momentum/class_5/event/py_bias": 0.4619043171405792, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.10684335231781, "metrics/particle/momentum/class_5/event/pz_bias": 0.06618919968605042, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6705543994903564, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34460780024528503, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.34661033749580383, "metrics/set_slot/angular_recall_all_dr0p10": 0.9512016177177429, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8400647044181824, "metrics/set_slot/soft_count_bias_mean": 0.17828507721424103}, "step_train_time": 6.453867197036743, "step_valid_time": 84066.54610586166, "step_total_time": 84072.99997329712, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003139007585482, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13057347196968816, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2883, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2864, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9524281466798811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543197616683218, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533730158730159, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04568023833167825, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461513049223654, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480304534922211, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9470899470899471, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9524281466798811, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020087966206943, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1286093547889694, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2131, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9434129089301503, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.951828724353256, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9476021314387212, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04817127564674398, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9420866489832007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9504906333630687, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9462699822380106, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9434129089301503, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996458358199586, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12339133476055242, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1198, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9324534161490683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9404855129209084, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9364522417153996, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05951448707909162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9301242236024845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9381362568519969, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9341130604288499, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9324534161490683} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_28000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..65ccb63635a0a486d9fe9475d90291fbf50ca4eb --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3265112753458403, "train": {"Classification_binary": 0.13121646642684937, "Cardinality": 0.10173005610704422, "Classification": 0.0731162577867508, "Regression_pt": 0.11155467480421066, "Regression_eta": 0.0014747269451618195, "Regression_sin_phi": 0.0009734013583511114, "Regression_cos_phi": 0.0008640731102786958, "Regression_energy": 0.11071474850177765, "HitGrouping": 0.15224143862724304, "Auxiliary": 0.17979039251804352, "Total": 0.90483558177948, "Total_unweighted": 0.4299143489333801}, "train_group": {}, "valid": {"Classification_binary": 0.15099000930786133, "Cardinality": 0.12430168191591899, "Classification": 0.07696129381656647, "Regression_pt": 0.11528733372688293, "Regression_eta": 0.001481108833104372, "Regression_sin_phi": 0.0008444997171560923, "Regression_cos_phi": 0.0008222126246740421, "Regression_energy": 0.11522356669108073, "HitGrouping": 0.1534770131111145, "Auxiliary": 0.19843673706054688, "Total": 0.9612389405568441, "Total_unweighted": 0.461610024717326, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8431469202041626, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5824030041694641, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3018796741962433, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5890116691589355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5910576581954956, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3534523546695709, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7898082137107849, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5229211449623108, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.29244324564933777, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5299609899520874, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5531837940216064, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43917316198349, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8587455749511719, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.573753833770752, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.29630202054977417, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5767278671264648, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5690655708312988, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.323933482170105, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8475426435470581, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5719775557518005, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2977507710456848, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5764859318733215, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5747179388999939, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34376898407936096, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026214491575956345, "metrics/hit_grouping/calorimeter/background_recall": 0.6434260010719299, "metrics/hit_grouping/calorimeter/completeness": 0.6425161361694336, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6504623889923096, "metrics/hit_grouping/calorimeter/energy_purity": 0.6429651975631714, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.34332871437072754, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8658609986305237, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5810913443565369, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23571215569972992, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5880598425865173, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5819981098175049, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2969244420528412, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8504619598388672, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5951229333877563, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.309073269367218, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5988208055496216, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5981956720352173, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3455846607685089, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8057259917259216, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5285670161247253, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4172670841217041, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5287126302719116, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5348421931266785, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.44103148579597473, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8323752284049988, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6060737371444702, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3486764132976532, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6134474277496338, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6224534511566162, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3881033957004547, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8216661810874939, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6771262884140015, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4521193206310272, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6752586960792542, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6807611584663391, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3856619596481323, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8671596050262451, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5218669176101685, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22324895858764648, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.524110734462738, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5098296403884888, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29074594378471375, "metrics/hit_grouping/calorimeter/purity": 0.6333293318748474, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8422314524650574, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1712903082370758, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11230769008398056, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.17083150148391724, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12346351891756058, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8438456058502197, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3165252208709717, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12428734451532364, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.31386303901672363, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.30236735939979553, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2291904240846634, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8478683829307556, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5952104330062866, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.31168022751808167, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6002370715141296, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6000005602836609, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3568664789199829, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7375139594078064, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41019126772880554, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5259939432144165, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5687764286994934, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5820382833480835, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8021973371505737, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3118339776992798, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5688467621803284, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5826425552368164, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.436343252658844, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6774609684944153, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.430652916431427, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44967007637023926, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4943060576915741, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6625186800956726, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7069470286369324, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41417911648750305, "metrics/hit_grouping/tracker/all/object_completeness": 0.4838308095932007, "metrics/hit_grouping/tracker/all/object_purity": 0.5252838134765625, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.618033230304718, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006283227354288101, "metrics/hit_grouping/tracker/background_recall": 0.04756263643503189, "metrics/hit_grouping/tracker/completeness": 0.49891629815101624, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4447510242462158, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7409828901290894, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3971754014492035, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5604830980300903, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5883487462997437, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5779498815536499, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.682999849319458, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43765467405319214, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4511316120624542, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5098923444747925, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6565771698951721, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6568359732627869, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42698219418525696, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3484160900115967, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.40164169669151306, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6660598516464233, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.706291913986206, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41984182596206665, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4815400540828705, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5389144420623779, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6104421019554138, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7034063339233398, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.40745672583580017, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45670217275619507, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5305249094963074, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6238349080085754, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7084230780601501, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.40798407793045044, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.49158528447151184, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5066049695014954, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6269023418426514, "metrics/hit_grouping/tracker/purity": 0.47621679306030273, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8902941346168518, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04517810419201851, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13477164506912231, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1385686695575714, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07862728089094162, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6952165365219116, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2737169563770294, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.36238807439804077, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3756032884120941, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5339553952217102, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6833689212799072, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4803140163421631, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.54509037733078, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5948753952980042, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7009031176567078, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5796855688095093, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.710774302482605, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8075237274169922, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6518412232398987, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09395826607942581, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12103652209043503, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.081851959228516, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30729803442955017, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17685824632644653, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8935694694519043, "metrics/particle/class_1/pid_accuracy": 0.9721662402153015, "metrics/particle/class_2/efficiency": 0.5356742143630981, "metrics/particle/class_2/pid_accuracy": 0.7631413340568542, "metrics/particle/class_3/efficiency": 0.76871258020401, "metrics/particle/class_3/pid_accuracy": 0.9737664461135864, "metrics/particle/class_4/efficiency": 0.7478344440460205, "metrics/particle/class_4/pid_accuracy": 0.5302445292472839, "metrics/particle/class_5/efficiency": 0.9419999718666077, "metrics/particle/class_5/pid_accuracy": 0.6326963901519775, "metrics/particle/count/bias_mean": -2.443359375, "metrics/particle/count/mae": 3.810546875, "metrics/particle/count/prediction_mean": 67.61263275146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.061301339417696, "metrics/particle/event/energy_response_mean": 0.9709377884864807, "metrics/particle/event/met_abs_error": 5.876828670501709, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04818923771381378, "metrics/particle/event/scalar_pt_response_mean": 0.9849331378936768, "metrics/particle/event/vector_pt_closure": 0.05143652856349945, "metrics/particle/matched/delta_eta_abs_mean": 0.014574545435607433, "metrics/particle/matched/delta_phi_abs_mean": 0.014830850064754486, "metrics/particle/matched/delta_r_mean": 0.023379741236567497, "metrics/particle/matched/energy_relative_abs_mean": 0.13639400899410248, "metrics/particle/matched/pid_accuracy": 0.9555714130401611, "metrics/particle/matched/pt_relative_abs_mean": 0.13567903637886047, "metrics/particle/matching/duplicate_fraction": 0.046151772141456604, "metrics/particle/matching/efficiency": 0.8086537718772888, "metrics/particle/matching/f1": 0.8230058550834656, "metrics/particle/matching/purity": 0.8378766179084778, "metrics/particle/matching/target_coverage_dr0p05": 0.7575878500938416, "metrics/particle/matching/target_coverage_dr0p10": 0.8533074259757996, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6323810815811157, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8426017165184021, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022931110113859177, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13384845852851868, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13277709484100342, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008251255378127098, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17159070074558258, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0074090962298214436, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6718498468399048, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.011293841525912285, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8813600540161133, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8773.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21489118039608002, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8623650074005127, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8649073839187622, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7148630023002625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11298006772994995, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.465194702148438, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014584345743060112, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9673798680305481, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04059210047125816, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9888131618499756, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10285.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8022128939628601, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.9268312454223633, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.142584800720215, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5931644439697266, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.022347452118992805, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.8570532202720642, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.018872261047363, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.6351909637451172, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.360330581665039, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023088470101356506, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1430896818637848, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14053450524806976, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011692713014781475, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1799163818359375, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02717127650976181, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.432178258895874, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07696735113859177, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.885261297225952, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2061.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3678371012210846, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.24664306640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2369415760040283, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.0819334983825684, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1569230705499649, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.49420928955078, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05269230902194977, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2791645526885986, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10609924048185349, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.139247894287109, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2600.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.682224452495575, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.7338924407958984, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.868751525878906, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2222671508789062, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0009435452520847321, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05849820375442505, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.715113162994385, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5774686336517334, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.69655990600586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02316521480679512, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1343761682510376, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13433212041854858, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.028352132067084312, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17488601803779602, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006218553753569722, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2239048331975937, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.017716223374009132, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7180094122886658, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17689.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18991923332214355, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1604474782943726, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8535756468772888, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7121127247810364, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11590996384620667, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.334508895874023, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004088110756129026, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3999359607696533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02936846949160099, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8103973865509033, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20792.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8251621127128601, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.9658162593841553, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.972294807434082, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.813569068908691, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8077494502067566, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.1178765296936035, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.4647693634033203, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6144484281539917, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.6023173332214355, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02308766357600689, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13484346866607666, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13430200517177582, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020965810865163803, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1742495596408844, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0046278443187475204, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4489888846874237, "metrics/particle/momentum/all/accepted/pz_bias": 0.020022189244627953, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5808653831481934, "metrics/particle/momentum/all/accepted/samples": 28523.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2112816721200943, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.076067328453064, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.963837742805481, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8187083601951599, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11818154901266098, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.559799194335938, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01104611437767744, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7183181047439575, "metrics/particle/momentum/all/assigned/pz_bias": 0.03872010484337807, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7679476737976074, "metrics/particle/momentum/all/assigned/samples": 33677.0, "metrics/particle/momentum/all/efficiency": 0.8058710694313049, "metrics/particle/momentum/all/event/energy_bias": -8.62653923034668, "metrics/particle/momentum/all/event/p3_error_mean": 16.886072158813477, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.604718208312988, "metrics/particle/momentum/all/event/px_bias": -0.8310403823852539, "metrics/particle/momentum/all/event/py_bias": 3.033428192138672, "metrics/particle/momentum/all/event/pz_abs_mean": 12.264713287353516, "metrics/particle/momentum/all/event/pz_bias": 0.5567260980606079, "metrics/particle/momentum/all/event/pz_residual_rms": 19.606409072875977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024307752028107643, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11442992836236954, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11525540798902512, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0060943677090108395, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15102575719356537, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004433100577443838, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.46159428358078003, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0017103690188378096, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5091702938079834, "metrics/particle/momentum/class_1/accepted/samples": 14888.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11071635782718658, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2679230272769928, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2622312605381012, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12821076810359955, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05568188801407814, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.3804941177368164, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007743838708847761, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5949810147285461, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.011129741556942463, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1910157203674316, "metrics/particle/momentum/class_1/assigned/samples": 16271.0, "metrics/particle/momentum/class_1/efficiency": 0.8921915292739868, "metrics/particle/momentum/class_1/event/energy_bias": 0.5231257677078247, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.246848106384277, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.951910972595215, "metrics/particle/momentum/class_1/event/px_bias": -1.0523393154144287, "metrics/particle/momentum/class_1/event/py_bias": 2.3765857219696045, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.407057762145996, "metrics/particle/momentum/class_1/event/pz_bias": -0.2390214204788208, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.138120651245117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02967202290892601, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1802043467760086, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18261970579624176, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.004135248716920614, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22407495975494385, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021451104432344437, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4394313097000122, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.12597380578517914, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.639389991760254, "metrics/particle/momentum/class_2/accepted/samples": 1585.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5287559628486633, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6349319815635681, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7714420557022095, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.45648711919784546, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.24123363196849823, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.201548099517822, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.043937474489212036, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3436241149902344, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.29097694158554077, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.265049457550049, "metrics/particle/momentum/class_2/assigned/samples": 2367.0, "metrics/particle/momentum/class_2/efficiency": 0.5324152112007141, "metrics/particle/momentum/class_2/event/energy_bias": -10.889825820922852, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.585359573364258, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.131875038146973, "metrics/particle/momentum/class_2/event/px_bias": 0.646486759185791, "metrics/particle/momentum/class_2/event/py_bias": -0.5798249244689941, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.975069046020508, "metrics/particle/momentum/class_2/event/pz_bias": 1.5264906883239746, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.234424591064453, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020625052973628044, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15450914204120636, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15177680552005768, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04306016489863396, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19292455911636353, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019778141286224127, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27124637365341187, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.028767595067620277, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0623446702957153, "metrics/particle/momentum/class_3/accepted/samples": 11629.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2703994810581207, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0555646419525146, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7806835174560547, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6505284309387207, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16695353388786316, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.3077335357666, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008881239220499992, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5681382417678833, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029029913246631622, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3524816036224365, "metrics/particle/momentum/class_3/assigned/samples": 14525.0, "metrics/particle/momentum/class_3/efficiency": 0.7647136449813843, "metrics/particle/momentum/class_3/event/energy_bias": 3.1578710079193115, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.825730323791504, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.7899885177612305, "metrics/particle/momentum/class_3/event/px_bias": -0.2283836305141449, "metrics/particle/momentum/class_3/event/py_bias": 0.8167773485183716, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.296942710876465, "metrics/particle/momentum/class_3/event/pz_bias": -0.7159532904624939, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.151535987854004, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026257134974002838, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14178583025932312, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1400814950466156, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0564388781785965, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18054991960525513, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007352941203862429, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8287012577056885, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3803260326385498, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0116491317749023, "metrics/particle/momentum/class_4/accepted/samples": 272.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3529815971851349, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3804354667663574, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3281177282333374, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2107936143875122, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19382022321224213, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.523075103759766, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014044944196939468, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1055711507797241, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2320571392774582, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2888286113739014, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.7472527623176575, "metrics/particle/momentum/class_4/event/energy_bias": -0.8406856060028076, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2858572006225586, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5898199081420898, "metrics/particle/momentum/class_4/event/px_bias": 0.014365814626216888, "metrics/particle/momentum/class_4/event/py_bias": -0.0017320141196250916, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3709696531295776, "metrics/particle/momentum/class_4/event/pz_bias": 0.2108885645866394, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.077799320220947, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017549142241477966, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14450359344482422, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1490384042263031, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0152271194383502, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1865444928407669, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0469798669219017, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8326512575149536, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6176285743713379, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.8034749031066895, "metrics/particle/momentum/class_5/accepted/samples": 149.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.057520508766174316, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17687903344631195, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18455222249031067, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00031678585219196975, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0569620244204998, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.26676854491233826, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.050632912665605545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.0045156478881836, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.44385629892349243, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1844482421875, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9371069073677063, "metrics/particle/momentum/class_5/event/energy_bias": -0.5770249962806702, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8719767332077026, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4390614032745361, "metrics/particle/momentum/class_5/event/px_bias": -0.21116989850997925, "metrics/particle/momentum/class_5/event/py_bias": 0.4216219186782837, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9426580667495728, "metrics/particle/momentum/class_5/event/pz_bias": -0.22567826509475708, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0508816242218018, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34710535407066345, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.34810635447502136, "metrics/set_slot/angular_recall_all_dr0p10": 0.9542033076286316, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8405200242996216, "metrics/set_slot/soft_count_bias_mean": -1.2224456071853638}, "step_train_time": 6.647748708724976, "step_valid_time": 87380.69268250465, "step_total_time": 87387.34043169022, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9935144810118072, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1277929201449819, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3017, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2868, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514370664023786, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954590652966523, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9530112508272667, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04540934703347697, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9474727452923687, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506131919124958, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490403706154864, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514370664023786, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9814534152669383, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1259816494247381, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2140, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2134, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9460654288240495, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954079358002675, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9500554938956715, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04592064199732501, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434129089301503, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514043691484618, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9473917869034406, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9460654288240495, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9896572606179976, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12608228572842417, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1200, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9316770186335404, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9345794392523364, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9331259720062208, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06542056074766354, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9277950310559007, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9306853582554517, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9292379471228616, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9316770186335404} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_29000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..063566172f99e5ee3d70808416668fcc7a2fc29d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3738866780367633, "train": {"Classification_binary": 0.13064044713974, "Cardinality": 0.10561281442642212, "Classification": 0.07404913008213043, "Regression_pt": 0.11428621411323547, "Regression_eta": 0.001394342165440321, "Regression_sin_phi": 0.0007911171414889395, "Regression_cos_phi": 0.00078319211024791, "Regression_energy": 0.11396293342113495, "HitGrouping": 0.15408159792423248, "Auxiliary": 0.18607355654239655, "Total": 0.8981865644454956, "Total_unweighted": 0.435907376173418}, "train_group": {}, "valid": {"Classification_binary": 0.15282156070073447, "Cardinality": 0.10870065291722615, "Classification": 0.0744361678759257, "Regression_pt": 0.11198122302691142, "Regression_eta": 0.0014732189786930878, "Regression_sin_phi": 0.0008195087624092897, "Regression_cos_phi": 0.0008326712995767593, "Regression_energy": 0.11133715510368347, "HitGrouping": 0.15399262309074402, "Auxiliary": 0.19244259595870972, "Total": 0.9354180494944254, "Total_unweighted": 0.4537015057479342, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8430593013763428, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5860705375671387, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2888448238372803, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5914773344993591, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5968424677848816, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3457069993019104, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7917913198471069, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5373509526252747, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.31175872683525085, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5440382361412048, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5586142539978027, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4266350269317627, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8529050350189209, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5708985328674316, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2838033139705658, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5737149715423584, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5764803290367126, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.330182820558548, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8442550897598267, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5727716684341431, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2879331409931183, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5767689347267151, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5814302563667297, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34382978081703186, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026504194363951683, "metrics/hit_grouping/calorimeter/background_recall": 0.6464734673500061, "metrics/hit_grouping/calorimeter/completeness": 0.6436604857444763, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6556096076965332, "metrics/hit_grouping/calorimeter/energy_purity": 0.6480802297592163, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32918259501457214, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8623649477958679, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5799936652183533, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2301582545042038, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5874590873718262, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5856618881225586, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.29083308577537537, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8477132320404053, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5982032418251038, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.30179524421691895, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6014384627342224, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6079702377319336, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3475254774093628, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8023188710212708, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.530983030796051, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.39577803015708923, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5284333229064941, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5448158383369446, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4521552324295044, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.831451952457428, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6076450943946838, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.33435067534446716, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6138795614242554, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6347528696060181, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.389349102973938, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8219462633132935, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6814683675765991, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44113031029701233, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6786078214645386, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6876729726791382, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38356882333755493, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8608739376068115, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5213714241981506, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21842774748802185, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5238409638404846, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.510509729385376, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2898084819316864, "metrics/hit_grouping/calorimeter/purity": 0.6346461176872253, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8097240328788757, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1586814820766449, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14000000059604645, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15726007521152496, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.11679109185934067, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8168960809707642, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3035861551761627, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11915621161460876, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3029610514640808, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.29506486654281616, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23945267498493195, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8465646505355835, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5970801115036011, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3007935583591461, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6014586091041565, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6078041791915894, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3561778664588928, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7354246973991394, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4024207890033722, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.520016074180603, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5692033767700195, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5874178409576416, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8025199770927429, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3079080283641815, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5717864632606506, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5771575570106506, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4256870448589325, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6742726564407349, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.429707795381546, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4452819526195526, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49818792939186096, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6683468818664551, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7044018507003784, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41082561016082764, "metrics/hit_grouping/tracker/all/object_completeness": 0.4795573651790619, "metrics/hit_grouping/tracker/all/object_purity": 0.5272846817970276, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6223313808441162, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004392158705741167, "metrics/hit_grouping/tracker/background_recall": 0.035834863781929016, "metrics/hit_grouping/tracker/completeness": 0.49448829889297485, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.43977510929107666, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7397873997688293, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3916172981262207, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.554311990737915, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5872346758842468, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.580501139163971, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6780199408531189, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4340376853942871, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.44892144203186035, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5106841921806335, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6632400751113892, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.653881311416626, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42880257964134216, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34616297483444214, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4118422865867615, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6717233061790466, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7050955295562744, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42307692766189575, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48021548986434937, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5440587997436523, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6143960952758789, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7015956044197083, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4174433946609497, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46268200874328613, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5356827974319458, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6205059885978699, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7039915919303894, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3937017023563385, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.48166733980178833, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5039557814598083, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6329899430274963, "metrics/hit_grouping/tracker/purity": 0.47109371423721313, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8934823870658875, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05169418081641197, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13545770943164825, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1364210844039917, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08297132700681686, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6911194920539856, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2768273651599884, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3536946475505829, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.37079066038131714, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5489891171455383, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6802213788032532, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47488340735435486, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5406462550163269, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5982159972190857, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7039726376533508, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5746342539787292, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6845394372940063, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7727166414260864, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6435238122940063, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0889069065451622, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.11271898448467255, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.065067291259766, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.300595760345459, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17015594244003296, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8951127529144287, "metrics/particle/class_1/pid_accuracy": 0.9721700549125671, "metrics/particle/class_2/efficiency": 0.5241193771362305, "metrics/particle/class_2/pid_accuracy": 0.7664811611175537, "metrics/particle/class_3/efficiency": 0.7870551347732544, "metrics/particle/class_3/pid_accuracy": 0.9741597771644592, "metrics/particle/class_4/efficiency": 0.744947075843811, "metrics/particle/class_4/pid_accuracy": 0.5077519416809082, "metrics/particle/class_5/efficiency": 0.9419999718666077, "metrics/particle/class_5/pid_accuracy": 0.656050980091095, "metrics/particle/count/bias_mean": -1.6119791269302368, "metrics/particle/count/mae": 3.5130207538604736, "metrics/particle/count/prediction_mean": 68.44400787353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06406395882368088, "metrics/particle/event/energy_response_mean": 0.9923317432403564, "metrics/particle/event/met_abs_error": 6.276514053344727, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051105547696352005, "metrics/particle/event/scalar_pt_response_mean": 1.0080829858779907, "metrics/particle/event/vector_pt_closure": 0.053634997457265854, "metrics/particle/matched/delta_eta_abs_mean": 0.014466017484664917, "metrics/particle/matched/delta_phi_abs_mean": 0.0149014201015234, "metrics/particle/matched/delta_r_mean": 0.023309115320444107, "metrics/particle/matched/energy_relative_abs_mean": 0.13440005481243134, "metrics/particle/matched/pid_accuracy": 0.9562742710113525, "metrics/particle/matched/pt_relative_abs_mean": 0.13510994613170624, "metrics/particle/matching/duplicate_fraction": 0.05252544581890106, "metrics/particle/matching/efficiency": 0.8163392543792725, "metrics/particle/matching/f1": 0.8258404731750488, "metrics/particle/matching/purity": 0.8355655074119568, "metrics/particle/matching/target_coverage_dr0p05": 0.7602642774581909, "metrics/particle/matching/target_coverage_dr0p10": 0.859292209148407, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6390535831451416, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.850742518901825, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022379057481884956, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1327410638332367, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13380752503871918, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024516506120562553, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1724325716495514, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006110670976340771, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6394838094711304, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03631212189793587, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7028728723526, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8837.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22930572926998138, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.070574402809143, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.17312753200531, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0306049585342407, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1207655519247055, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.080608367919922, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014641148038208485, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9641958475112915, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.043403059244155884, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9012463092803955, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10450.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8080651164054871, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.6100975275039673, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.60988712310791, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.9460532665252686, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5098221302032471, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.31819862127304077, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.395679473876953, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.073760747909546, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.917168617248535, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02235446684062481, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1339912712574005, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13938508927822113, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04109843820333481, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18044504523277283, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023164119571447372, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3417658805847168, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.026870589703321457, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.744147777557373, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2029.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3797551989555359, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8504738211631775, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.5979052782058716, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.465118408203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.16923077404499054, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 10.349194526672363, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05538461357355118, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2771620750427246, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0317908376455307, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.579318523406982, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2600.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.671631932258606, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.2363996505737305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.490886688232422, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.202695608139038, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0828118845820427, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.024144772440195084, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.347175598144531, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3284893035888672, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.70331382751465, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023428766056895256, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13552352786064148, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13498087227344513, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01606287993490696, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1742122620344162, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007787728682160378, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22500179708003998, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011363796889781952, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7349587678909302, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17977.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18343394994735718, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1789493560791016, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8720344305038452, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7202227711677551, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11105824261903763, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.390531539916992, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004615530837327242, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.41430023312568665, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.021764537319540977, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8699063062667847, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21016.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8385968208312988, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7881081700325012, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.580286026000977, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.218989372253418, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5788528919219971, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9510489106178284, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.844170570373535, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5450863242149353, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.045318126678467, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023031583055853844, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13456323742866516, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13493120670318604, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020414086058735847, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17411604523658752, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003987102769315243, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.43055227398872375, "metrics/particle/momentum/all/accepted/pz_bias": -0.020098382607102394, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4870343208312988, "metrics/particle/momentum/all/accepted/samples": 28843.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21248917281627655, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.12063467502594, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0197973251342773, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8722869753837585, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11847589910030365, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.817071914672852, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011565783992409706, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7251629829406738, "metrics/particle/momentum/all/assigned/pz_bias": -0.024314839392900467, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.836008071899414, "metrics/particle/momentum/all/assigned/samples": 34066.0, "metrics/particle/momentum/all/efficiency": 0.8149121403694153, "metrics/particle/momentum/all/event/energy_bias": -0.8381941318511963, "metrics/particle/momentum/all/event/p3_error_mean": 19.463537216186523, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.098621368408203, "metrics/particle/momentum/all/event/px_bias": -1.1714868545532227, "metrics/particle/momentum/all/event/py_bias": -1.2933924198150635, "metrics/particle/momentum/all/event/pz_abs_mean": 14.944952011108398, "metrics/particle/momentum/all/event/pz_bias": -3.947336435317993, "metrics/particle/momentum/all/event/pz_residual_rms": 23.934673309326172, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024737190455198288, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11509108543395996, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11621136963367462, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.024300141260027885, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15225118398666382, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0034740781411528587, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44139933586120605, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.013454969972372055, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.408555030822754, "metrics/particle/momentum/class_1/accepted/samples": 14968.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1102832779288292, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3045923411846161, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3197134733200073, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20205625891685486, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.058220431208610535, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.8911585807800293, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00714024156332016, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5802668929100037, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.010969685390591621, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1057682037353516, "metrics/particle/momentum/class_1/assigned/samples": 16386.0, "metrics/particle/momentum/class_1/efficiency": 0.896985650062561, "metrics/particle/momentum/class_1/event/energy_bias": 5.317392349243164, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.565742492675781, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.590787887573242, "metrics/particle/momentum/class_1/event/px_bias": -0.9637445211410522, "metrics/particle/momentum/class_1/event/py_bias": -1.075834035873413, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.245111465454102, "metrics/particle/momentum/class_1/event/pz_bias": -1.1728923320770264, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.40203094482422, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028918210417032242, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17928466200828552, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1801355928182602, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.00803422462195158, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2196654975414276, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020208604633808136, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4087892770767212, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.019095463678240776, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5237245559692383, "metrics/particle/momentum/class_2/accepted/samples": 1534.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5928335189819336, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7096832394599915, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7987315654754639, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4708453118801117, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27564895153045654, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.2403368949890137, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04697157070040703, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3417601585388184, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08239366114139557, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.3860764503479, "metrics/particle/momentum/class_2/assigned/samples": 2427.0, "metrics/particle/momentum/class_2/efficiency": 0.5152838230133057, "metrics/particle/momentum/class_2/event/energy_bias": -7.385148525238037, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.194350242614746, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.248093128204346, "metrics/particle/momentum/class_2/event/px_bias": 0.2434900999069214, "metrics/particle/momentum/class_2/event/py_bias": 0.052235037088394165, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.470855712890625, "metrics/particle/momentum/class_2/event/pz_bias": -1.2867499589920044, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.128175735473633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02015022560954094, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1528329700231552, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15223978459835052, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016221066936850548, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19182585179805756, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00201376061886549, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2641710638999939, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.030989177525043488, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.99704909324646, "metrics/particle/momentum/class_3/accepted/samples": 11918.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2628372013568878, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0692248344421387, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8125609159469604, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6610486507415771, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15841248631477356, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.156576156616211, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010176390409469604, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5960730314254761, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.052822280675172806, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5415759086608887, "metrics/particle/momentum/class_3/assigned/samples": 14740.0, "metrics/particle/momentum/class_3/efficiency": 0.7837180495262146, "metrics/particle/momentum/class_3/event/energy_bias": 2.2039709091186523, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.006267547607422, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.392412185668945, "metrics/particle/momentum/class_3/event/px_bias": -0.13586263358592987, "metrics/particle/momentum/class_3/event/py_bias": -0.750028133392334, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.111165523529053, "metrics/particle/momentum/class_3/event/pz_bias": -1.4019131660461426, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.189037322998047, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025180954486131668, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14883652329444885, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14651131629943848, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07253183424472809, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19072429835796356, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007246376946568489, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7837023735046387, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17934216558933258, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.786108374595642, "metrics/particle/momentum/class_4/accepted/samples": 276.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3090163767337799, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6249964237213135, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2965688705444336, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1767563819885254, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.19382022321224213, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.484607696533203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014044944196939468, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1015424728393555, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.022301234304904938, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.479827880859375, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.7582417726516724, "metrics/particle/momentum/class_4/event/energy_bias": -0.87641441822052, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.394951820373535, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6641160249710083, "metrics/particle/momentum/class_4/event/px_bias": 0.0006851665675640106, "metrics/particle/momentum/class_4/event/py_bias": 0.11988402158021927, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4365767240524292, "metrics/particle/momentum/class_4/event/pz_bias": 0.008866088464856148, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.38112211227417, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017501596361398697, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14257971942424774, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1442868858575821, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.003993337973952293, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1815117448568344, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.040816325694322586, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9440677165985107, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.5970446467399597, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9618842601776123, "metrics/particle/momentum/class_5/accepted/samples": 147.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05424947291612625, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17332856357097626, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1805010437965393, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01854698173701763, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0573248416185379, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2598208487033844, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05095541477203369, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1237668991088867, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3948393762111664, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.243954658508301, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.9245283007621765, "metrics/particle/momentum/class_5/event/energy_bias": -0.0979950875043869, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.2155489921569824, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5960618257522583, "metrics/particle/momentum/class_5/event/px_bias": -0.31605497002601624, "metrics/particle/momentum/class_5/event/py_bias": 0.36035072803497314, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2221754789352417, "metrics/particle/momentum/class_5/event/pz_bias": -0.09464692324399948, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.760739326477051, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34424248337745667, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3457045257091522, "metrics/set_slot/angular_recall_all_dr0p10": 0.9558946490287781, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8436704277992249, "metrics/set_slot/soft_count_bias_mean": -0.5353373885154724}, "step_train_time": 5.838029623031616, "step_valid_time": 90517.77339243889, "step_total_time": 90523.61142230034, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0074120747203086, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1283839347195923, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2875, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2861, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9497852659398744, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9541984732824428, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9519867549668874, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04580152671755725, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9451602246448629, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495519415864587, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9473509933774834, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9497852659398744, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0065601095472274, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1311236164455194, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2149, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2143, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9500442086648984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538393253439857, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.951937984496124, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0461606746560142, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9473916887709991, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511762094984465, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.949280177187154, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9500442086648984, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.017631455683218, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1306363535429227, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1272, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9324534161490683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9441823899371069, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.93828125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05581761006289308, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9293478260869565, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9410377358490566, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.93515625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9324534161490683} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_3000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..99e27f471ffeef291b6de360e2b33fc5e9dcc59f --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14212620807276863, "train": {"Classification_binary": 0.3585509657859802, "Cardinality": 0.25472140312194824, "Classification": 0.24805578589439392, "Regression_pt": 0.3792414665222168, "Regression_eta": 0.005139496177434921, "Regression_sin_phi": 0.0026967807207256556, "Regression_cos_phi": 0.0028189602307975292, "Regression_energy": 0.37539663910865784, "HitGrouping": 0.24978670477867126, "Auxiliary": 0.4576839506626129, "Total": 2.4248969554901123, "Total_unweighted": 1.3719000944402069}, "train_group": {}, "valid": {"Classification_binary": 0.3665351867675781, "Cardinality": 0.22247578700383505, "Classification": 0.270504355430603, "Regression_pt": 0.4011048873265584, "Regression_eta": 0.005199491046369076, "Regression_sin_phi": 0.0026917398596803346, "Regression_cos_phi": 0.002670749711493651, "Regression_energy": 0.3960295120875041, "HitGrouping": 0.2552340030670166, "Auxiliary": 0.471196969350179, "Total": 2.445526917775472, "Total_unweighted": 1.4447359222297866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7884011268615723, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2297152727842331, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35449135303497314, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.22732390463352203, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2219855785369873, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4993860423564911, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7628317475318909, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.22963008284568787, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3510674238204956, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.22803615033626556, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2031124234199524, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5221958756446838, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7978518605232239, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.25005042552948, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3540956377983093, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.24880212545394897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.23828944563865662, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4798574447631836, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7916694283485413, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.24167507886886597, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3539513647556305, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.24002793431282043, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2298877239227295, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.48993921279907227, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025222068652510643, "metrics/hit_grouping/calorimeter/background_recall": 0.623173177242279, "metrics/hit_grouping/calorimeter/completeness": 0.34120088815689087, "metrics/hit_grouping/calorimeter/energy_completeness": 0.36389562487602234, "metrics/hit_grouping/calorimeter/energy_purity": 0.3595641851425171, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5527979135513306, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8271443843841553, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.23313312232494354, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26915496587753296, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2333136647939682, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.2179657369852066, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4147506654262543, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7732597589492798, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2672092914581299, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38573509454727173, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.2647653818130493, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.26411697268486023, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5321446061134338, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7357650995254517, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.233146533370018, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5003160238265991, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.22846117615699768, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.21945267915725708, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6051068305969238, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7511195540428162, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2683797776699066, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4381812512874603, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.26718977093696594, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2667069137096405, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5797570943832397, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6986956000328064, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4280123710632324, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6797488331794739, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.41988900303840637, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.427705317735672, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6473050713539124, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8471604585647583, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.1891215443611145, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22170884907245636, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.1878116875886917, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.16499142348766327, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.37324225902557373, "metrics/hit_grouping/calorimeter/purity": 0.3359799385070801, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9058912992477417, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05472424253821373, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13384614884853363, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05489743873476982, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012816229835152626, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03076923079788685, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8718022108078003, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.048924416303634644, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08494868874549866, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.049432601779699326, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.030432062223553658, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2514253258705139, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7846260070800781, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.25674599409103394, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37411463260650635, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2549360990524292, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.24598628282546997, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5134443640708923, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6302863359451294, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.417812317609787, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.2519240379333496, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2615358531475067, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7425283789634705, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7291878461837769, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.41278743743896484, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3322208821773529, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2784624397754669, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5709478259086609, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5591344237327576, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42104434967041016, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.20975565910339355, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.22743375599384308, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.8016066551208496, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5959450602531433, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41932740807533264, "metrics/hit_grouping/tracker/all/object_completeness": 0.23339755833148956, "metrics/hit_grouping/tracker/all/object_purity": 0.24251005053520203, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7635083794593811, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01259695552289486, "metrics/hit_grouping/tracker/background_recall": 0.057572707533836365, "metrics/hit_grouping/tracker/completeness": 0.24896615743637085, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5804427266120911, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.5946916937828064, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4064692556858063, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.26902905106544495, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.29312533140182495, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7703872323036194, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5828470587730408, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.422615647315979, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.20915661752223969, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.21878713369369507, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7829811573028564, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.612643837928772, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44437703490257263, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.18005655705928802, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.15535660088062286, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7275485396385193, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6125510931015015, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4608195424079895, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.23792846500873566, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2418898046016693, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7552120685577393, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.647558867931366, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5233022570610046, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.30289536714553833, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.29362615942955017, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6997336745262146, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5652361512184143, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.34699133038520813, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.21527467668056488, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2343299388885498, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7855303287506104, "metrics/hit_grouping/tracker/purity": 0.23915830254554749, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8794750571250916, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01824500411748886, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.028267377987504005, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.024528052657842636, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6431790590286255, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.14930015802383423, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.08080319315195084, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.07126210629940033, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6111975312232971, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.552039623260498, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.504574716091156, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.27864664793014526, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.291629433631897, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8740924596786499, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.449448823928833, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.348158061504364, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.358918696641922, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5435162782669067, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03627854585647583, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.012711478397250175, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.06865882873535, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.17121070623397827, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.04077088087797165, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.4862683117389679, "metrics/particle/class_1/pid_accuracy": 0.879638671875, "metrics/particle/class_2/efficiency": 0.34081220626831055, "metrics/particle/class_2/pid_accuracy": 0.18696510791778564, "metrics/particle/class_3/efficiency": 0.4278634786605835, "metrics/particle/class_3/pid_accuracy": 0.886833131313324, "metrics/particle/class_4/efficiency": 0.4456207752227783, "metrics/particle/class_4/pid_accuracy": 0.20950323343276978, "metrics/particle/class_5/efficiency": 0.4659999907016754, "metrics/particle/class_5/pid_accuracy": 0.19313304126262665, "metrics/particle/count/bias_mean": -9.168620109558105, "metrics/particle/count/mae": 10.276692390441895, "metrics/particle/count/prediction_mean": 60.88737106323242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1778271645307541, "metrics/particle/event/energy_response_mean": 1.0199946165084839, "metrics/particle/event/met_abs_error": 18.72234535217285, "metrics/particle/event/scalar_pt_relative_abs_error": 0.16478049755096436, "metrics/particle/event/scalar_pt_response_mean": 1.0729267597198486, "metrics/particle/event/vector_pt_closure": 0.14112773537635803, "metrics/particle/matched/delta_eta_abs_mean": 0.022496551275253296, "metrics/particle/matched/delta_phi_abs_mean": 0.021861737594008446, "metrics/particle/matched/delta_r_mean": 0.034990180283784866, "metrics/particle/matched/energy_relative_abs_mean": 0.20150475203990936, "metrics/particle/matched/pid_accuracy": 0.8292576670646667, "metrics/particle/matched/pt_relative_abs_mean": 0.19869430363178253, "metrics/particle/matching/duplicate_fraction": 0.11650609970092773, "metrics/particle/matching/efficiency": 0.44843223690986633, "metrics/particle/matching/f1": 0.479831337928772, "metrics/particle/matching/purity": 0.5159586668014526, "metrics/particle/matching/target_coverage_dr0p05": 0.3918740451335907, "metrics/particle/matching/target_coverage_dr0p10": 0.5495418310165405, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.2812017798423767, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5247848629951477, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03473491966724396, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20332691073417664, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20102962851524353, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007144062779843807, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24294427037239075, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.019309161230921745, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.4008853435516357, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.1760558784008026, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.9947891235351562, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4661.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.7192572951316833, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.9082305431365967, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.019209861755371, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.8365867137908936, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.38937851786613464, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 25.763511657714844, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.042598869651556015, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.274435043334961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.3680435121059418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.913891792297363, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8850.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.426207035779953, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 7.214153289794922, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 19.870086669921875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.958639144897461, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.0065468549728394, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 2.0690572261810303, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 16.885419845581055, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 6.031698703765869, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 26.913663864135742, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.032568614929914474, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20532554388046265, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.2021590620279312, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003985957708209753, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2440502792596817, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.048152294009923935, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.5329606533050537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0848798081278801, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.174896240234375, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 893.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9977855086326599, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.01078724861145, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.008965015411377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.842467308044434, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.43169400095939636, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.852542877197266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08579234778881073, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.7669317722320557, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.4382624328136444, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.94888973236084, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1830.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.29559749364852905, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -14.611320495605469, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 18.173412322998047, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.6941306591033936, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.5016962289810181, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.43336617946624756, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 17.879804611206055, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.0064897537231445, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 36.94014358520508, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03532731533050537, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20030444860458374, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19833967089653015, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008601518347859383, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23937420547008514, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0018227167893201113, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4593696892261505, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.04992052912712097, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1546857357025146, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 10424.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6329472064971924, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.086690425872803, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.425044059753418, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.246062755584717, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3905019760131836, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.68641471862793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.017301037907600403, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.188745379447937, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.16363021731376648, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.4342050552368164, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19941.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.48626208305358887, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 19.00381088256836, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 23.117860794067383, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 19.451030731201172, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -6.087159633636475, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 6.079370498657227, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.292783737182617, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.6738107204437256, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 16.349761962890625, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03500032797455788, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20146676898002625, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19933784008026123, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007918398827314377, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24068373441696167, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00951308012008667, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8499143123626709, "metrics/particle/momentum/all/accepted/pz_bias": 0.0886697769165039, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.2322280406951904, "metrics/particle/momentum/all/accepted/samples": 15978.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6796960830688477, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.622032880783081, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.7511909008026123, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.5719032287597656, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.39263904094696045, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.59330177307129, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.028705790638923645, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.6566084623336792, "metrics/particle/momentum/all/assigned/pz_bias": 0.23912201821804047, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.409649848937988, "metrics/particle/momentum/all/assigned/samples": 30621.0, "metrics/particle/momentum/all/efficiency": 0.4514324367046356, "metrics/particle/momentum/all/event/energy_bias": 11.606644630432129, "metrics/particle/momentum/all/event/p3_error_mean": 46.600830078125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 23.950786590576172, "metrics/particle/momentum/all/event/px_bias": -7.595403671264648, "metrics/particle/momentum/all/event/py_bias": 8.581793785095215, "metrics/particle/momentum/all/event/pz_abs_mean": 34.158817291259766, "metrics/particle/momentum/all/event/pz_bias": 10.711997985839844, "metrics/particle/momentum/all/event/pz_residual_rms": 48.385169982910156, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04376688599586487, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2186994105577469, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21536511182785034, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.011079292744398117, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.25470829010009766, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.011946848593652248, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.0081161260604858, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.14173582196235657, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.375620126724243, "metrics/particle/momentum/class_1/accepted/samples": 8203.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5127427577972412, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.9269182682037354, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.086310625076294, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.8727930784225464, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.35392117500305176, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.94513988494873, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.025599349290132523, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.6076576709747314, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.3449331521987915, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.837739944458008, "metrics/particle/momentum/class_1/assigned/samples": 14766.0, "metrics/particle/momentum/class_1/efficiency": 0.4915802776813507, "metrics/particle/momentum/class_1/event/energy_bias": 41.42937469482422, "metrics/particle/momentum/class_1/event/p3_error_mean": 35.76319885253906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 19.826980590820312, "metrics/particle/momentum/class_1/event/px_bias": -2.968106985092163, "metrics/particle/momentum/class_1/event/py_bias": 6.3146071434021, "metrics/particle/momentum/class_1/event/pz_abs_mean": 25.512836456298828, "metrics/particle/momentum/class_1/event/pz_bias": 15.344049453735352, "metrics/particle/momentum/class_1/event/pz_residual_rms": 35.78850555419922, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04060981050133705, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22971181571483612, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22468815743923187, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06278760731220245, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26537081599235535, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02822580561041832, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7521398067474365, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.012510146014392376, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.069540500640869, "metrics/particle/momentum/class_2/accepted/samples": 992.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7154208421707153, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7777226567268372, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1051865816116333, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7481891512870789, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3481556475162506, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.039030075073242, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05507832393050194, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.638005256652832, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.21792058646678925, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.56439733505249, "metrics/particle/momentum/class_2/assigned/samples": 1979.0, "metrics/particle/momentum/class_2/efficiency": 0.3332213759422302, "metrics/particle/momentum/class_2/event/energy_bias": -28.8290958404541, "metrics/particle/momentum/class_2/event/p3_error_mean": 30.676305770874023, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.544299125671387, "metrics/particle/momentum/class_2/event/px_bias": -1.6783504486083984, "metrics/particle/momentum/class_2/event/py_bias": -1.2105369567871094, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.3730411529541, "metrics/particle/momentum/class_2/event/pz_bias": -3.950202703475952, "metrics/particle/momentum/class_2/event/pz_residual_rms": 36.6961555480957, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02301902137696743, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17561040818691254, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17517992854118347, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04418409615755081, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21713033318519592, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002752293599769473, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.48173221945762634, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03955507278442383, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4730188846588135, "metrics/particle/momentum/class_3/accepted/samples": 6540.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8588318824768066, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.885803699493408, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.967292785644531, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.854457855224609, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4421287775039673, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 29.077695846557617, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.027242276817560196, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.531554102897644, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11842988431453705, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.707808971405029, "metrics/particle/momentum/class_3/assigned/samples": 13435.0, "metrics/particle/momentum/class_3/efficiency": 0.4300650954246521, "metrics/particle/momentum/class_3/event/energy_bias": 4.955812454223633, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.887279510498047, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.628838539123535, "metrics/particle/momentum/class_3/event/px_bias": -2.0631041526794434, "metrics/particle/momentum/class_3/event/py_bias": 1.575952172279358, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.767681121826172, "metrics/particle/momentum/class_3/event/pz_bias": -0.7103050351142883, "metrics/particle/momentum/class_3/event/pz_residual_rms": 27.45819091796875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03774160146713257, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1894625872373581, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1926310658454895, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.031282227486371994, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23928120732307434, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.7373195886611938, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2288426011800766, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.3607563972473145, "metrics/particle/momentum/class_4/accepted/samples": 168.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7639487385749817, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.483173847198486, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.057001113891602, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.862579345703125, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4079754650592804, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.837438583374023, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0460122711956501, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.3066844940185547, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.4407136142253876, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.465614318847656, "metrics/particle/momentum/class_4/assigned/samples": 326.0, "metrics/particle/momentum/class_4/efficiency": 0.4615384638309479, "metrics/particle/momentum/class_4/event/energy_bias": -2.2152419090270996, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.589909076690674, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.3655221462249756, "metrics/particle/momentum/class_4/event/px_bias": -0.23013244569301605, "metrics/particle/momentum/class_4/event/py_bias": 1.1780081987380981, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.52311635017395, "metrics/particle/momentum/class_4/event/pz_bias": 0.10885076224803925, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.138717651367188, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.040606483817100525, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22465230524539948, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2326679676771164, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.053595200181007385, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2731136381626129, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02666666731238365, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.731109857559204, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.40825092792510986, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.276325225830078, "metrics/particle/momentum/class_5/accepted/samples": 75.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.3351733386516571, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4782458543777466, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4559048116207123, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009388030506670475, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.30434781312942505, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6424250602722168, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09565217047929764, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.8201112747192383, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5463495850563049, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.929083824157715, "metrics/particle/momentum/class_5/assigned/samples": 115.0, "metrics/particle/momentum/class_5/efficiency": 0.4716981053352356, "metrics/particle/momentum/class_5/event/energy_bias": -3.734203577041626, "metrics/particle/momentum/class_5/event/p3_error_mean": 5.255858421325684, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.167659759521484, "metrics/particle/momentum/class_5/event/px_bias": -0.6557086706161499, "metrics/particle/momentum/class_5/event/py_bias": 0.7237633466720581, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.514091730117798, "metrics/particle/momentum/class_5/event/pz_bias": -0.08039330691099167, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.099797248840332, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.688696563243866, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6938401460647583, "metrics/set_slot/angular_recall_all_dr0p10": 0.8197591304779053, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5847443342208862, "metrics/set_slot/soft_count_bias_mean": 0.06902668625116348}, "step_train_time": 5.27994179725647, "step_valid_time": 8972.027152061462, "step_total_time": 8977.307094097137, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1772936243519712, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3659736256877901, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2979, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2482, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2090, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8199537495870499, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8331654917757637, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8265068265068265, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16683450822423632, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6904525933267261, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7015777106411547, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6959706959706959, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8199537495870499, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1046293866098151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3306183862592392, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2228, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1863, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1701, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8236074270557029, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8361759425493717, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8298440979955457, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16382405745062836, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7519893899204244, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7634649910233393, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7576837416481069, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8236074270557029, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0592100249191487, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3258077609594662, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1213, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1006, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 932, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.781055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8293487221764221, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8044782087165134, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17065127782357792, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7236024844720497, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7683429513602638, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7453018792483007, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.781055900621118} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_30000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..6d8b4809bef0dc46e411094573f36ab0affaca0c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4212620807276861, "train": {"Classification_binary": 0.13422861695289612, "Cardinality": 0.10404424369335175, "Classification": 0.0702524483203888, "Regression_pt": 0.10383138805627823, "Regression_eta": 0.0012247380800545216, "Regression_sin_phi": 0.0007376945577561855, "Regression_cos_phi": 0.0007232562638819218, "Regression_energy": 0.10338950157165527, "HitGrouping": 0.15452131628990173, "Auxiliary": 0.1730038970708847, "Total": 0.8609036207199097, "Total_unweighted": 0.41438764380291104}, "train_group": {}, "valid": {"Classification_binary": 0.15319323539733887, "Cardinality": 0.11044296622276306, "Classification": 0.07452469070752461, "Regression_pt": 0.1102148989836375, "Regression_eta": 0.0013845028976599376, "Regression_sin_phi": 0.0008190263373156389, "Regression_cos_phi": 0.0008328016847372055, "Regression_energy": 0.10941253105799358, "HitGrouping": 0.1526913046836853, "Auxiliary": 0.19235467910766602, "Total": 0.9316928386688232, "Total_unweighted": 0.45038168706620735, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8447234034538269, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5892919301986694, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3008406460285187, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5947174429893494, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5959093570709229, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3407953083515167, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7890750169754028, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5419929027557373, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3032870292663574, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5495404601097107, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5680214762687683, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43171805143356323, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8604529500007629, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5740892291069031, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2850428819656372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5770300626754761, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.573165237903595, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31391385197639465, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8489890098571777, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5761024355888367, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.29176291823387146, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5802559852600098, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5800227522850037, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33313068747520447, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026594819501042366, "metrics/hit_grouping/calorimeter/background_recall": 0.6469669938087463, "metrics/hit_grouping/calorimeter/completeness": 0.6459411382675171, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6575700044631958, "metrics/hit_grouping/calorimeter/energy_purity": 0.6501219868659973, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3311102092266083, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8638784885406494, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5849296450614929, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23009853065013885, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5921761393547058, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5881922245025635, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.288504034280777, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8536598086357117, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5958713293075562, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.302886962890625, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6005740761756897, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6023085117340088, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3366084396839142, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8126049637794495, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5368172526359558, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4106939733028412, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5338516235351562, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5395068526268005, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4239666163921356, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8371139168739319, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6127794981002808, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3440672755241394, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6198811531066895, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6310344338417053, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3738399147987366, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8245962262153625, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6900262832641602, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44008371233940125, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6850754022598267, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.688431441783905, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3689168095588684, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8648766279220581, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5220943093299866, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21655283868312836, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5242434740066528, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5113096833229065, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2847864031791687, "metrics/hit_grouping/calorimeter/purity": 0.636954128742218, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8241416811943054, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.15506955981254578, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1246153861284256, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15170511603355408, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.1183166652917862, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.032307691872119904, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8248302936553955, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3175962567329407, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11687571555376053, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.31757256388664246, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.30246081948280334, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23888255655765533, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.850908100605011, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5999445915222168, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3053843080997467, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6044986844062805, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6058278679847717, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34496328234672546, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7480839490890503, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39465034008026123, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5356237888336182, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5828822255134583, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5629109144210815, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8029109239578247, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.31575995683670044, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5707833766937256, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5785419344902039, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4245653450489044, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6867347955703735, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.41883909702301025, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4596448540687561, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5031436681747437, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6471607685089111, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7159096598625183, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4025127589702606, "metrics/hit_grouping/tracker/all/object_completeness": 0.49301961064338684, "metrics/hit_grouping/tracker/all/object_purity": 0.5346969366073608, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6017853617668152, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005935514345765114, "metrics/hit_grouping/tracker/background_recall": 0.05194574594497681, "metrics/hit_grouping/tracker/completeness": 0.5060423016548157, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4322434663772583, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7502226233482361, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3889749348163605, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5703694820404053, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5921231508255005, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5588154792785645, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6926828622817993, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41709500551223755, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.46077826619148254, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.524515688419342, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6371597051620483, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6644178628921509, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4170711934566498, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.355569988489151, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4180360436439514, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6595873832702637, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7162379622459412, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40240833163261414, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49350330233573914, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5560895800590515, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5930984616279602, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7077246904373169, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.43608522415161133, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.47107332944869995, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5368918776512146, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6058588624000549, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7169211506843567, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.396745502948761, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.49624842405319214, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5064417719841003, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6123858690261841, "metrics/hit_grouping/tracker/purity": 0.4828495979309082, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8900011777877808, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05473501235246658, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1390710026025772, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13964417576789856, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07906168699264526, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6971808671951294, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27060651779174805, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.36196595430374146, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.38881388306617737, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5308449864387512, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6943860054016113, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46478956937789917, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5560379028320312, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6049891114234924, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6808925271034241, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5872883796691895, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7040072679519653, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7978201508522034, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6549906134605408, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10156102478504181, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12418586760759354, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.952070236206055, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29632219672203064, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.165882408618927, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9067273736000061, "metrics/particle/class_1/pid_accuracy": 0.9738576412200928, "metrics/particle/class_2/efficiency": 0.5463315844535828, "metrics/particle/class_2/pid_accuracy": 0.7492812871932983, "metrics/particle/class_3/efficiency": 0.7943063378334045, "metrics/particle/class_3/pid_accuracy": 0.9745847582817078, "metrics/particle/class_4/efficiency": 0.7459095120429993, "metrics/particle/class_4/pid_accuracy": 0.4929032325744629, "metrics/particle/class_5/efficiency": 0.9380000233650208, "metrics/particle/class_5/pid_accuracy": 0.6353944540023804, "metrics/particle/count/bias_mean": -0.875, "metrics/particle/count/mae": 3.3216145038604736, "metrics/particle/count/prediction_mean": 69.18099212646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06390459835529327, "metrics/particle/event/energy_response_mean": 0.9679841995239258, "metrics/particle/event/met_abs_error": 6.0446248054504395, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050739824771881104, "metrics/particle/event/scalar_pt_response_mean": 0.9813452363014221, "metrics/particle/event/vector_pt_closure": 0.04969463869929314, "metrics/particle/matched/delta_eta_abs_mean": 0.013783286325633526, "metrics/particle/matched/delta_phi_abs_mean": 0.014792225323617458, "metrics/particle/matched/delta_r_mean": 0.022647731006145477, "metrics/particle/matched/energy_relative_abs_mean": 0.13128197193145752, "metrics/particle/matched/pid_accuracy": 0.9558927416801453, "metrics/particle/matched/pt_relative_abs_mean": 0.13052210211753845, "metrics/particle/matching/duplicate_fraction": 0.053330447524785995, "metrics/particle/matching/efficiency": 0.8267661929130554, "metrics/particle/matching/f1": 0.8319617509841919, "metrics/particle/matching/purity": 0.8372231125831604, "metrics/particle/matching/target_coverage_dr0p05": 0.7719365358352661, "metrics/particle/matching/target_coverage_dr0p10": 0.8650354146957397, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6586993336677551, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8606676459312439, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02206394262611866, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12844711542129517, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12787356972694397, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0005890150205232203, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16738727688789368, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007126948796212673, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6448279023170471, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05283078923821449, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8710163831710815, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8980.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22458745539188385, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9344651103019714, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0196166038513184, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8597111105918884, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1114063411951065, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.950922966003418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01470867358148098, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9387021064758301, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06531910598278046, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7121939659118652, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10538.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8211411833763123, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.94517183303833, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.424569129943848, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6973090171813965, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.25012168288230896, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2945426404476166, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.260575771331787, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7546166777610779, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.686739921569824, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022549718618392944, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.137588769197464, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13335658609867096, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0013288117479532957, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17289017140865326, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024061597883701324, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3898435831069946, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.017206436023116112, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.0887579917907715, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2078.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4075443148612976, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1534785032272339, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2689638137817383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.0975677967071533, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.16244804859161377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.588380813598633, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05326785147190094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.501389980316162, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.04746689274907112, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.308351516723633, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2647.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6878517270088196, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.0491838455200195, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.18038558959961, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2220624685287476, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0049490295350551605, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03851918876171112, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.032173156738281, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.35590827465057373, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.580516815185547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022528113797307014, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13152846693992615, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13056787848472595, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0033334584441035986, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17022690176963806, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006049606599844992, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22203843295574188, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008740084245800972, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6933035254478455, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18183.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1773691028356552, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9536640048027039, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7907611727714539, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6320925354957581, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10499098896980286, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.399378776550293, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00360565516166389, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.38029032945632935, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006047635339200497, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5881222486495972, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21078.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8482064008712769, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.3256022930145264, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.8864426612854, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.691619873046875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.33398178219795227, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.35400259494781494, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.481795072555542, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.22353853285312653, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.792270660400391, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022387100383639336, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13101284205913544, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12993863224983215, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0017975326627492905, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1695515215396881, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004274819511920214, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4348679780960083, "metrics/particle/momentum/all/accepted/pz_bias": 0.020436596125364304, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.600633144378662, "metrics/particle/momentum/all/accepted/samples": 29241.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2096739411354065, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9631959795951843, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9753475189208984, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8153151869773865, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1114029735326767, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.979898452758789, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010857192799448967, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7159029841423035, "metrics/particle/momentum/all/assigned/pz_bias": 0.020036352798342705, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0243732929229736, "metrics/particle/momentum/all/assigned/samples": 34263.0, "metrics/particle/momentum/all/efficiency": 0.8261569738388062, "metrics/particle/momentum/all/event/energy_bias": -9.319958686828613, "metrics/particle/momentum/all/event/p3_error_mean": 18.263328552246094, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.655996322631836, "metrics/particle/momentum/all/event/px_bias": 0.5791545510292053, "metrics/particle/momentum/all/event/py_bias": 0.6100260615348816, "metrics/particle/momentum/all/event/pz_abs_mean": 13.892007827758789, "metrics/particle/momentum/all/event/pz_bias": 0.6222468614578247, "metrics/particle/momentum/all/event/pz_residual_rms": 23.633302688598633, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023906363174319267, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10801967233419418, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10775227099657059, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.008250718005001545, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1429470330476761, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003957783803343773, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.43560633063316345, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.036153394728899, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4475468397140503, "metrics/particle/momentum/class_1/accepted/samples": 15160.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1086784303188324, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.25355103611946106, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30893558263778687, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16738423705101013, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05241421237587929, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.9941108226776123, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006984512787312269, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5681406259536743, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.062214743345975876, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.471714735031128, "metrics/particle/momentum/class_1/assigned/samples": 16465.0, "metrics/particle/momentum/class_1/efficiency": 0.9084916114807129, "metrics/particle/momentum/class_1/event/energy_bias": 0.22563162446022034, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.624709129333496, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.421004295349121, "metrics/particle/momentum/class_1/event/px_bias": -0.40958523750305176, "metrics/particle/momentum/class_1/event/py_bias": -0.2551349699497223, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.417644500732422, "metrics/particle/momentum/class_1/event/pz_bias": 1.930029034614563, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.903430938720703, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029385995119810104, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1812066286802292, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18253189325332642, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.011189837008714676, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22278651595115662, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020257826894521713, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4857722520828247, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.039520736783742905, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7509500980377197, "metrics/particle/momentum/class_2/accepted/samples": 1629.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5829675197601318, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6683445572853088, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8551017045974731, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5150682926177979, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25410163402557373, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.094690322875977, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04401760548353195, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.355137348175049, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.014421636238694191, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.89428186416626, "metrics/particle/momentum/class_2/assigned/samples": 2499.0, "metrics/particle/momentum/class_2/efficiency": 0.5471951365470886, "metrics/particle/momentum/class_2/event/energy_bias": -9.975811958312988, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.977273941040039, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.017908573150635, "metrics/particle/momentum/class_2/event/px_bias": 1.1026091575622559, "metrics/particle/momentum/class_2/event/py_bias": -0.3612757921218872, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.501428604125977, "metrics/particle/momentum/class_2/event/pz_bias": -0.8407976627349854, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.846479415893555, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01949075423181057, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15248394012451172, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15018457174301147, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.015401976183056831, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19007191061973572, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019976694602519274, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.264367014169693, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.002905926201492548, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1826802492141724, "metrics/particle/momentum/class_3/accepted/samples": 12014.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25914594531059265, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8006973266601562, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7390788793563843, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5879020690917969, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15293560922145844, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.640392303466797, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009334415197372437, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5848249197006226, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.025156637653708458, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.8545806407928467, "metrics/particle/momentum/class_3/assigned/samples": 14784.0, "metrics/particle/momentum/class_3/efficiency": 0.7900308966636658, "metrics/particle/momentum/class_3/event/energy_bias": 2.1099655628204346, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.356915473937988, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.765990257263184, "metrics/particle/momentum/class_3/event/px_bias": -0.210089311003685, "metrics/particle/momentum/class_3/event/py_bias": 0.6972556114196777, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.914124011993408, "metrics/particle/momentum/class_3/event/pz_bias": -0.6425038576126099, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.535618782043457, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026777120307087898, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15382377803325653, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1440262645483017, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.027500495314598083, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18816052377223969, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0139860138297081, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8439847230911255, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.22370319068431854, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.3487250804901123, "metrics/particle/momentum/class_4/accepted/samples": 286.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27806052565574646, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.418121099472046, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.2790937423706055, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1526424884796143, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15126051008701324, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.8570170402526855, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014005602337419987, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0126233100891113, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.102619469165802, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.196807384490967, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.7857142686843872, "metrics/particle/momentum/class_4/event/energy_bias": -1.4918899536132812, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3835039138793945, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6703195571899414, "metrics/particle/momentum/class_4/event/px_bias": 0.1559196561574936, "metrics/particle/momentum/class_4/event/py_bias": 0.16634389758110046, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.378365397453308, "metrics/particle/momentum/class_4/event/pz_bias": 0.08331010490655899, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.354879856109619, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01651827059686184, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14636285603046417, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1523531824350357, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.019515134394168854, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19570843875408173, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8050960302352905, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.2891152501106262, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4873406887054443, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04652335122227669, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1853996366262436, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.17486150562763214, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03146068751811981, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.25402408838272095, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7816529273986816, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2444247007369995, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.438188076019287, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": -0.18785303831100464, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0899338722229004, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5492218732833862, "metrics/particle/momentum/class_5/event/px_bias": -0.05969971790909767, "metrics/particle/momentum/class_5/event/py_bias": 0.3628372550010681, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1221843957901, "metrics/particle/momentum/class_5/event/pz_bias": 0.09220919013023376, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6998696327209473, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.33667638897895813, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.33814340829849243, "metrics/set_slot/angular_recall_all_dr0p10": 0.9567496180534363, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8490511775016785, "metrics/set_slot/soft_count_bias_mean": 0.8073851466178894}, "step_train_time": 5.177862167358398, "step_valid_time": 2813.9939630031586, "step_total_time": 2819.171825647354, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9847139184282073, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1235355181092328, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2883, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2860, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9524281466798811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543197616683218, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533730158730159, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04568023833167825, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9448298645523621, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.946706388613042, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9457671957671958, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9524281466798811, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9793862802453215, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12138520247553986, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2148, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2142, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9496021220159151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580731489741302, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9538188277087034, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04192685102586976, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.946949602122016, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553969669937555, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511545293072824, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9496021220159151, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9804205181709953, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11754103077830946, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1274, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1208, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.937888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9481946624803768, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9430132708821234, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05180533751962323, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355590062111802, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.945839874411303, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9406713505074161, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.937888198757764} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_31000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..34b4a209f552900b7ba7b44a1c574cad4b6c79e7 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4686374834186091, "train": {"Classification_binary": 0.11914480477571487, "Cardinality": 0.09389054775238037, "Classification": 0.06280076503753662, "Regression_pt": 0.10956273972988129, "Regression_eta": 0.0012533154804259539, "Regression_sin_phi": 0.000776261615101248, "Regression_cos_phi": 0.0007466168608516455, "Regression_energy": 0.10881936550140381, "HitGrouping": 0.14890119433403015, "Auxiliary": 0.1667785793542862, "Total": 0.8307607173919678, "Total_unweighted": 0.40310386900091544}, "train_group": {}, "valid": {"Classification_binary": 0.14647761980692545, "Cardinality": 0.10427931944529216, "Classification": 0.07290886839230855, "Regression_pt": 0.10345715284347534, "Regression_eta": 0.001498738459000985, "Regression_sin_phi": 0.0008213223579029242, "Regression_cos_phi": 0.0008339549725254377, "Regression_energy": 0.10267989834149678, "HitGrouping": 0.15033937493960062, "Auxiliary": 0.18598655859629312, "Total": 0.9054818153381348, "Total_unweighted": 0.4286775551736355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8470702171325684, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5904729962348938, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.29224520921707153, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5953580141067505, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5984587669372559, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34107866883277893, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.797951340675354, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5387395024299622, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30193156003952026, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5473242402076721, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5607364773750305, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4229075014591217, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8618413805961609, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5792486071586609, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.28003305196762085, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5828883647918701, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5810068845748901, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.312106192111969, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8513573408126831, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5792269706726074, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.28592705726623535, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5837110280990601, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5848045945167542, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3313677906990051, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026282088831067085, "metrics/hit_grouping/calorimeter/background_recall": 0.6430373787879944, "metrics/hit_grouping/calorimeter/completeness": 0.6504260897636414, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6597093939781189, "metrics/hit_grouping/calorimeter/energy_purity": 0.6520955562591553, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3272923231124878, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8689899444580078, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5889314413070679, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22944162786006927, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5968367457389832, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5907590985298157, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2835473418235779, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8555994629859924, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6043485403060913, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2962154150009155, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6077650189399719, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6087061762809753, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33236292004585266, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8096143007278442, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5325066447257996, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.39476677775382996, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5308619141578674, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5472932457923889, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4315510094165802, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8378193378448486, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6155268549919128, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.33366551995277405, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6225705742835999, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6368054747581482, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3722827732563019, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8308945894241333, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6906051635742188, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.42700156569480896, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6878948211669922, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.687050998210907, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.36368393898010254, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8685301542282104, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5259498953819275, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21655283868312836, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5286030173301697, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5158165693283081, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2832462787628174, "metrics/hit_grouping/calorimeter/purity": 0.6411706805229187, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8262009024620056, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1561966836452484, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.127692312002182, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15389011800289154, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12104307860136032, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.029230769723653793, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8415528535842896, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30388882756233215, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10661345720291138, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3036482632160187, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.29646727442741394, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2291904240846634, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8524575233459473, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6040797233581543, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2996130585670471, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6089804172515869, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6112732291221619, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3436844050884247, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.748510479927063, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3971906900405884, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5276820063591003, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5668010711669922, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.560669481754303, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8046497702598572, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3213684856891632, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5649357438087463, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5744231939315796, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43241727352142334, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6863116025924683, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4178939759731293, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.45385175943374634, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49650341272354126, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6525163650512695, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7159371376037598, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4032212495803833, "metrics/hit_grouping/tracker/all/object_completeness": 0.48654279112815857, "metrics/hit_grouping/tracker/all/object_purity": 0.525285005569458, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.604949951171875, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004023095592856407, "metrics/hit_grouping/tracker/background_recall": 0.03097790665924549, "metrics/hit_grouping/tracker/completeness": 0.5015113949775696, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4385729134082794, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7485113739967346, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38387244939804077, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5611972212791443, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5903209447860718, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5652847290039062, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6950898766517639, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4224252700805664, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4578627943992615, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5074899196624756, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6411574482917786, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6657770872116089, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4257686138153076, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35129252076148987, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39982128143310547, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6545307636260986, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7159113883972168, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4115743935108185, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4847775101661682, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5347995758056641, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.598220705986023, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7211160659790039, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4487350285053253, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48011261224746704, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.518722653388977, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5872170329093933, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7150601148605347, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3843362331390381, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.48997315764427185, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5140438079833984, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6168344616889954, "metrics/hit_grouping/tracker/purity": 0.47760751843452454, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8987094163894653, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.053866203874349594, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1355045586824417, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13161377608776093, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07602085173130035, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7063806056976318, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2851218283176422, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3711681365966797, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.38542959094047546, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5241057276725769, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.692189633846283, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46414026618003845, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.547380268573761, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5947030782699585, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.686028003692627, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5904055237770081, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.709998607635498, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8042287230491638, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6501532793045044, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10467816144227982, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.11934850364923477, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.68060302734375, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3016083538532257, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17116853594779968, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9034823775291443, "metrics/particle/class_1/pid_accuracy": 0.9712890386581421, "metrics/particle/class_2/efficiency": 0.544761061668396, "metrics/particle/class_2/pid_accuracy": 0.765238881111145, "metrics/particle/class_3/efficiency": 0.796280026435852, "metrics/particle/class_3/pid_accuracy": 0.9761295318603516, "metrics/particle/class_4/efficiency": 0.7391722798347473, "metrics/particle/class_4/pid_accuracy": 0.5911458134651184, "metrics/particle/class_5/efficiency": 0.9559999704360962, "metrics/particle/class_5/pid_accuracy": 0.6589958071708679, "metrics/particle/count/bias_mean": -1.43359375, "metrics/particle/count/mae": 3.2565104961395264, "metrics/particle/count/prediction_mean": 68.62239837646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05764615535736084, "metrics/particle/event/energy_response_mean": 0.9826511740684509, "metrics/particle/event/met_abs_error": 6.07713508605957, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04645508527755737, "metrics/particle/event/scalar_pt_response_mean": 0.9889678955078125, "metrics/particle/event/vector_pt_closure": 0.0491904616355896, "metrics/particle/matched/delta_eta_abs_mean": 0.015328478999435902, "metrics/particle/matched/delta_phi_abs_mean": 0.014717352576553822, "metrics/particle/matched/delta_r_mean": 0.023794351145625114, "metrics/particle/matched/energy_relative_abs_mean": 0.12844397127628326, "metrics/particle/matched/pid_accuracy": 0.9570887088775635, "metrics/particle/matched/pt_relative_abs_mean": 0.12876805663108826, "metrics/particle/matching/duplicate_fraction": 0.04885013774037361, "metrics/particle/matching/efficiency": 0.8259855508804321, "metrics/particle/matching/f1": 0.8345242142677307, "metrics/particle/matching/purity": 0.8432412147521973, "metrics/particle/matching/target_coverage_dr0p05": 0.7625318169593811, "metrics/particle/matching/target_coverage_dr0p10": 0.864013135433197, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6620169878005981, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8609650135040283, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023482611402869225, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12652744352817535, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1266152262687683, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014824097976088524, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16591599583625793, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006044324953109026, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6172513961791992, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05182603374123573, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6987135410308838, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8934.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21326154470443726, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8290731906890869, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9157510995864868, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7777698040008545, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10881621390581131, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.43822193145752, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01300439890474081, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9118659496307373, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.07341166585683823, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8329131603240967, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10458.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8169348835945129, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5089124441146851, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.991092681884766, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.573549509048462, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1432005763053894, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.24676303565502167, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.889194965362549, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.8689061403274536, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.514288902282715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023179112002253532, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13524167239665985, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13843964040279388, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04417239874601364, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17798694968223572, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024661509320139885, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3106142282485962, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.07873455435037613, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4880712032318115, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2068.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3963172137737274, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2251497507095337, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.89853835105896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7687879800796509, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.16233766078948975, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.048011779785156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.055767763406038284, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2486605644226074, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08257116377353668, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.027198314666748, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2618.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6845415234565735, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.1154816150665283, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.411461353302002, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0606049299240112, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04893617331981659, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1512836515903473, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.29192590713501, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.065760374069214, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.249874114990234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02374713122844696, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12686485052108765, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12663781642913818, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0038129405584186316, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16574367880821228, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004956219927407801, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21492962539196014, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03298342972993851, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6814310550689697, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18159.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1831771582365036, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0305888652801514, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7901442050933838, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6359590888023376, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10403498262166977, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.295073509216309, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034218905493617058, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3654199242591858, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04664865508675575, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.60025155544281, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21041.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8470867872238159, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.41700729727745056, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.352095603942871, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.941402912139893, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.02620673179626465, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.2544761896133423, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.864325523376465, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8150497674942017, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.0364508628845215, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023625807836651802, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12735554575920105, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12746785581111908, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010048565454781055, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16669408977031708, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003909330815076828, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4158904552459717, "metrics/particle/momentum/all/accepted/pz_bias": -0.04200071841478348, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4268981218338013, "metrics/particle/momentum/all/accepted/samples": 29161.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2087545245885849, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9837473630905151, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.913700520992279, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7663574814796448, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10997449606657028, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.38344669342041, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010376058518886566, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6774360537528992, "metrics/particle/momentum/all/assigned/pz_bias": -0.0576089546084404, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6128478050231934, "metrics/particle/momentum/all/assigned/samples": 34117.0, "metrics/particle/momentum/all/efficiency": 0.8238967061042786, "metrics/particle/momentum/all/event/energy_bias": -4.041401386260986, "metrics/particle/momentum/all/event/p3_error_mean": 17.629640579223633, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.548789978027344, "metrics/particle/momentum/all/event/px_bias": 0.12047113478183746, "metrics/particle/momentum/all/event/py_bias": -1.6525230407714844, "metrics/particle/momentum/all/event/pz_abs_mean": 13.294419288635254, "metrics/particle/momentum/all/event/pz_bias": -4.749716758728027, "metrics/particle/momentum/all/event/pz_residual_rms": 20.239742279052734, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025717701762914658, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10500112175941467, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10540349781513214, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.013808856718242168, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14094951748847961, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0035801895428448915, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.41846853494644165, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02506115287542343, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.331315517425537, "metrics/particle/momentum/class_1/accepted/samples": 15083.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11926306039094925, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26687899231910706, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2778809666633606, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16027453541755676, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05338533595204353, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.569470167160034, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0065208119340240955, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5399792790412903, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.013222684152424335, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9067708253860474, "metrics/particle/momentum/class_1/assigned/samples": 16409.0, "metrics/particle/momentum/class_1/efficiency": 0.9038772583007812, "metrics/particle/momentum/class_1/event/energy_bias": 3.6811957359313965, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.040254592895508, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.001043319702148, "metrics/particle/momentum/class_1/event/px_bias": 0.09639228880405426, "metrics/particle/momentum/class_1/event/py_bias": -0.13325214385986328, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.730989456176758, "metrics/particle/momentum/class_1/event/pz_bias": -0.4111308455467224, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.618675231933594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029144177213311195, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17549210786819458, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1754418909549713, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0018153541022911668, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21397912502288818, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021250000223517418, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.402235984802246, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1432463377714157, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6039395332336426, "metrics/particle/momentum/class_2/accepted/samples": 1600.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5587016940116882, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6573259234428406, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7674680948257446, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4533712863922119, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25798526406288147, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.9720003604888916, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.047911547124385834, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.332477331161499, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18736600875854492, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.2476348876953125, "metrics/particle/momentum/class_2/assigned/samples": 2442.0, "metrics/particle/momentum/class_2/efficiency": 0.537453830242157, "metrics/particle/momentum/class_2/event/energy_bias": -10.068026542663574, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.774002075195312, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.369000434875488, "metrics/particle/momentum/class_2/event/px_bias": 0.1739075481891632, "metrics/particle/momentum/class_2/event/py_bias": -1.0667331218719482, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.995159149169922, "metrics/particle/momentum/class_2/event/pz_bias": -2.651357889175415, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.521760940551758, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020227113738656044, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1484566330909729, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1483377069234848, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0058681112714111805, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18742676079273224, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001576763461343944, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2594001591205597, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.048457544296979904, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8806146383285522, "metrics/particle/momentum/class_3/accepted/samples": 12050.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.250346302986145, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8106924295425415, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.637237787246704, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.483373999595642, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14777657389640808, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.412959098815918, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008202278055250645, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5374407172203064, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.08185530453920364, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1998887062072754, "metrics/particle/momentum/class_3/assigned/samples": 14752.0, "metrics/particle/momentum/class_3/efficiency": 0.79239821434021, "metrics/particle/momentum/class_3/event/energy_bias": 3.1740000247955322, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.760869979858398, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.6288838386535645, "metrics/particle/momentum/class_3/event/px_bias": 0.07302022725343704, "metrics/particle/momentum/class_3/event/py_bias": -0.617834210395813, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.3196306228637695, "metrics/particle/momentum/class_3/event/pz_bias": -1.8560187816619873, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.527737617492676, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027761027216911316, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14599502086639404, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1416560262441635, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.062097515910863876, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1876371055841446, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7084843516349792, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10573790222406387, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7840150594711304, "metrics/particle/momentum/class_4/accepted/samples": 273.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.27211087942123413, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.328904151916504, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5465134382247925, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4394692182540894, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1774647831916809, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.686322212219238, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0251319408416748, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.07982952147722244, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6993958950042725, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.75, "metrics/particle/momentum/class_4/event/energy_bias": -0.4106557369232178, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4702930450439453, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.667672038078308, "metrics/particle/momentum/class_4/event/px_bias": -0.04190627112984657, "metrics/particle/momentum/class_4/event/py_bias": -0.04292868822813034, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5385812520980835, "metrics/particle/momentum/class_4/event/pz_bias": 0.21458497643470764, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.649755477905273, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020038500428199768, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13249383866786957, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13187503814697266, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -7.494572491850704e-05, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1698375940322876, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025806451216340065, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6339032649993896, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.40351322293281555, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.2576799392700195, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06938877701759338, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.25161057710647583, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.23441453278064728, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.09431606531143188, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.01886792480945587, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.2163195610046387, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025157231837511063, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6566728353500366, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.34627050161361694, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.257484197616577, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9748427867889404, "metrics/particle/momentum/class_5/event/energy_bias": -0.41791418194770813, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8186973333358765, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3935751914978027, "metrics/particle/momentum/class_5/event/px_bias": -0.1809426248073578, "metrics/particle/momentum/class_5/event/py_bias": 0.20822520554065704, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9275628328323364, "metrics/particle/momentum/class_5/event/pz_bias": -0.04579384624958038, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.052994728088379, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3409877121448517, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3429867923259735, "metrics/set_slot/angular_recall_all_dr0p10": 0.9557645320892334, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8486794233322144, "metrics/set_slot/soft_count_bias_mean": -0.44442880153656006}, "step_train_time": 4.698152303695679, "step_valid_time": 5834.967508792877, "step_total_time": 5839.665661811829, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9901409207303828, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11963073363491872, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2888, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9540799471423852, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9521925486317178, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9531353135313532, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04780745136828223, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9497852659398744, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9479063633366304, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488448844884488, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9540799471423852, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9895862417921556, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11664124938408582, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2139, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9456233421750663, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9544846050870147, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.950033311125916, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04551539491298527, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9438549955791335, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9526996876394467, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9482567177437264, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9456233421750663, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9830865143822471, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1171811811717891, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1203, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9340062111801242, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9405785770132916, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9372808726139462, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05942142298670837, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9316770186335404, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9382329945269742, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9349435138293728, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9340062111801242} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_32000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..4a0b7921b3a4a4e128f22760bbf274509e46a25a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.5160128861095319, "train": {"Classification_binary": 0.13212667405605316, "Cardinality": 0.09486226737499237, "Classification": 0.0663890391588211, "Regression_pt": 0.11325925588607788, "Regression_eta": 0.0013426980003714561, "Regression_sin_phi": 0.0006964225322008133, "Regression_cos_phi": 0.0007714194944128394, "Regression_energy": 0.11194341629743576, "HitGrouping": 0.15529072284698486, "Auxiliary": 0.17628833651542664, "Total": 0.8648980259895325, "Total_unweighted": 0.426528925425373}, "train_group": {}, "valid": {"Classification_binary": 0.14808388551076254, "Cardinality": 0.10484179854393005, "Classification": 0.07110296189785004, "Regression_pt": 0.10604747136433919, "Regression_eta": 0.0014206475267807643, "Regression_sin_phi": 0.0008010473102331161, "Regression_cos_phi": 0.0007823524841417869, "Regression_energy": 0.10525975624720256, "HitGrouping": 0.15098129709561667, "Auxiliary": 0.18795510133107504, "Total": 0.9043291409810384, "Total_unweighted": 0.43349812234131, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8441362380981445, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5980928540229797, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.30017945170402527, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6027475595474243, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6079727411270142, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3403230309486389, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7882482409477234, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5445269346237183, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32056930661201477, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5523583292961121, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5565025210380554, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4205354154109955, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8576337099075317, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5820221900939941, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.28328683972358704, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5856994986534119, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.586928129196167, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32228076457977295, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8470667600631714, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5838303565979004, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.29206687211990356, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5881949067115784, "metrics/hit_grouping/calorimeter/all/object_purity": 0.590971052646637, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33689969778060913, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02642892114818096, "metrics/hit_grouping/calorimeter/background_recall": 0.6454494595527649, "metrics/hit_grouping/calorimeter/completeness": 0.6523399353027344, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6641982197761536, "metrics/hit_grouping/calorimeter/energy_purity": 0.6565428376197815, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3260113298892975, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8634735941886902, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5932305455207825, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23607046902179718, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6002142429351807, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5945753455162048, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28683188557624817, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8509758710861206, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6081695556640625, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2975497245788574, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6124353408813477, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6200697422027588, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33903443813323975, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8082651495933533, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5385697484016418, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.40487927198410034, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5374928712844849, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5530186891555786, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4406522512435913, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8355200290679932, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6194913983345032, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3350980877876282, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6269146800041199, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6451320052146912, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3786982297897339, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8303432464599609, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6926257014274597, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4395604431629181, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6891565322875977, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.687872588634491, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3699633777141571, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8616201281547546, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5315707921981812, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2269318401813507, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5336493253707886, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5203447341918945, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28773269057273865, "metrics/hit_grouping/calorimeter/purity": 0.6431542634963989, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8121715188026428, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.16587723791599274, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1230769231915474, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1648278385400772, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12155900150537491, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04461538419127464, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8192188739776611, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3349129557609558, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1214367151260376, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3340495526790619, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3294622004032135, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2428734302520752, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8494122624397278, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6070554852485657, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3054826855659485, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6118360161781311, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6160170435905457, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34853750467300415, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7374035716056824, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3921099901199341, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5238122344017029, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5809452533721924, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5854752063751221, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7965714931488037, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2910824418067932, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.568231999874115, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5951180458068848, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4391475021839142, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6809525489807129, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4178939759731293, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.45479118824005127, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5084460377693176, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6551153659820557, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7085322737693787, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3990648090839386, "metrics/hit_grouping/tracker/all/object_completeness": 0.48616063594818115, "metrics/hit_grouping/tracker/all/object_purity": 0.5386605262756348, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6149159073829651, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006240525748580694, "metrics/hit_grouping/tracker/background_recall": 0.062488894909620285, "metrics/hit_grouping/tracker/completeness": 0.49973630905151367, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.42995327711105347, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.745388925075531, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37703871726989746, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5610958933830261, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5986074805259705, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5689293742179871, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.686923623085022, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4214734435081482, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45461368560791016, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.516559362411499, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6537216901779175, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6496747732162476, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4241504967212677, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35333335399627686, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4290689527988434, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6757686138153076, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7057280540466309, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40519410371780396, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4854446351528168, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5557504892349243, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6091840267181396, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6936209797859192, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4527297019958496, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4700033366680145, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5304190516471863, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.636484682559967, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7148074507713318, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3816436529159546, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.48993444442749023, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.517845869064331, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6185904741287231, "metrics/hit_grouping/tracker/purity": 0.47697898745536804, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8946783542633057, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.047784533351659775, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1418258547782898, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.14358243346214294, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07428323477506638, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6976750493049622, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.268014520406723, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.36541983485221863, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.39413002133369446, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5438050627708435, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6844744086265564, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4617200791835785, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.546698272228241, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6088021397590637, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6964759826660156, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5769423842430115, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7176757454872131, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8142924904823303, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6577905416488647, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09121503680944443, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12698575854301453, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.822206497192383, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3137686252593994, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18332883715629578, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.911317765712738, "metrics/particle/class_1/pid_accuracy": 0.9758565425872803, "metrics/particle/class_2/efficiency": 0.5411711931228638, "metrics/particle/class_2/pid_accuracy": 0.7788142561912537, "metrics/particle/class_3/efficiency": 0.7805976867675781, "metrics/particle/class_3/pid_accuracy": 0.9712801575660706, "metrics/particle/class_4/efficiency": 0.751684308052063, "metrics/particle/class_4/pid_accuracy": 0.5441741347312927, "metrics/particle/class_5/efficiency": 0.9520000219345093, "metrics/particle/class_5/pid_accuracy": 0.6218487620353699, "metrics/particle/count/bias_mean": -1.623046875, "metrics/particle/count/mae": 3.3209636211395264, "metrics/particle/count/prediction_mean": 68.43294525146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.06117859482765198, "metrics/particle/event/energy_response_mean": 0.9804821610450745, "metrics/particle/event/met_abs_error": 6.068063735961914, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04790487885475159, "metrics/particle/event/scalar_pt_response_mean": 0.9944755434989929, "metrics/particle/event/vector_pt_closure": 0.05019057169556618, "metrics/particle/matched/delta_eta_abs_mean": 0.014389783143997192, "metrics/particle/matched/delta_phi_abs_mean": 0.014554237946867943, "metrics/particle/matched/delta_r_mean": 0.02297171764075756, "metrics/particle/matched/energy_relative_abs_mean": 0.1283866912126541, "metrics/particle/matched/pid_accuracy": 0.9575261473655701, "metrics/particle/matched/pt_relative_abs_mean": 0.12925967574119568, "metrics/particle/matching/duplicate_fraction": 0.05093565955758095, "metrics/particle/matching/efficiency": 0.8226771950721741, "metrics/particle/matching/f1": 0.8323186635971069, "metrics/particle/matching/purity": 0.8421888947486877, "metrics/particle/matching/target_coverage_dr0p05": 0.7707098126411438, "metrics/particle/matching/target_coverage_dr0p10": 0.8655465245246887, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6560693383216858, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8553054928779602, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022176256403326988, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12752075493335724, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1278209239244461, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022569820284843445, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1674739420413971, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006938227452337742, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6259602308273315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0692393034696579, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7632558345794678, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8936.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20398886501789093, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7084330916404724, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7495383024215698, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6160831451416016, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10982382297515869, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.46162748336792, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013787820935249329, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9240037202835083, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05461478978395462, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8114075660705566, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10444.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8171177506446838, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5314276218414307, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.44255256652832, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.388319730758667, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1529897004365921, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.18798144161701202, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.323727607727051, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.08039341866970062, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.94714069366455, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021929847076535225, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1322145015001297, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13479670882225037, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.035324256867170334, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1752990037202835, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020270269364118576, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2736173868179321, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.021657517179846764, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5550193786621094, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2072.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.37764719128608704, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9067814350128174, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.79147207736969, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.6524085998535156, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1542879045009613, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.233612060546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.050918836146593094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.213780641555786, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.012929603457450867, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.843541622161865, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2612.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.685865581035614, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.758033752441406, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.175589561462402, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1732112169265747, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.010520696640014648, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1126721054315567, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.011482238769531, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4722992777824402, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.029951095581055, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023139072582125664, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12655283510684967, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12751862406730652, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.022850114852190018, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1671987622976303, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004990849993191659, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21380940079689026, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.034115683287382126, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6518193483352661, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18033.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1766834855079651, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9014060497283936, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7391211986541748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6045842170715332, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10928492993116379, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.8787522315979, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030965651385486126, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3606260120868683, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02068033069372177, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.452949047088623, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20991.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8412091135978699, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.02732892334461212, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.378808975219727, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.228814125061035, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.0969775915145874, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0882411003112793, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.209057569503784, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8016068935394287, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.3513102531433105, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022756535559892654, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12725460529327393, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1281309276819229, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023653864860534668, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1678740680217743, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0038910505827516317, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4162440001964569, "metrics/particle/momentum/all/accepted/pz_bias": 0.04403446242213249, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4567745923995972, "metrics/particle/momentum/all/accepted/samples": 29041.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20047688484191895, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8426235914230347, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8230503797531128, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6884979605674744, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11290275305509567, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.047677993774414, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010044937953352928, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6756128072738647, "metrics/particle/momentum/all/assigned/pz_bias": 0.03049519471824169, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.705383539199829, "metrics/particle/momentum/all/assigned/samples": 34047.0, "metrics/particle/momentum/all/efficiency": 0.8205062747001648, "metrics/particle/momentum/all/event/energy_bias": -5.262132167816162, "metrics/particle/momentum/all/event/p3_error_mean": 17.330570220947266, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.140554428100586, "metrics/particle/momentum/all/event/px_bias": -0.26048803329467773, "metrics/particle/momentum/all/event/py_bias": -1.3888945579528809, "metrics/particle/momentum/all/event/pz_abs_mean": 13.054567337036133, "metrics/particle/momentum/all/event/pz_bias": 0.4097011089324951, "metrics/particle/momentum/all/event/pz_residual_rms": 24.030014038085938, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02445046976208687, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10475266724824905, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10511785000562668, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014678983017802238, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14067748188972473, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003421277739107609, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.41898784041404724, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.018406257033348083, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.364988088607788, "metrics/particle/momentum/class_1/accepted/samples": 15199.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09967389702796936, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23660464584827423, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.257364958524704, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.14642059803009033, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04969586431980133, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.2639517784118652, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005474452394992113, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5205587148666382, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012502552010118961, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7672513723373413, "metrics/particle/momentum/class_1/assigned/samples": 16440.0, "metrics/particle/momentum/class_1/efficiency": 0.9108287692070007, "metrics/particle/momentum/class_1/event/energy_bias": 6.058665752410889, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.239091873168945, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.493891716003418, "metrics/particle/momentum/class_1/event/px_bias": -0.869945228099823, "metrics/particle/momentum/class_1/event/py_bias": -0.051039695739746094, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.868606567382812, "metrics/particle/momentum/class_1/event/pz_bias": -0.5093684792518616, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.26395606994629, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0283957626670599, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17236575484275818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17517134547233582, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0003059987793676555, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.214326873421669, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022944550961256027, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4036959409713745, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.10258309543132782, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6164236068725586, "metrics/particle/momentum/class_2/accepted/samples": 1569.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5647963881492615, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6534791588783264, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8201828002929688, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5076290965080261, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2519587576389313, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.512819290161133, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04907216504216194, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3583176136016846, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1081029400229454, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.009427547454834, "metrics/particle/momentum/class_2/assigned/samples": 2425.0, "metrics/particle/momentum/class_2/efficiency": 0.5270406603813171, "metrics/particle/momentum/class_2/event/energy_bias": -9.35909652709961, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.553291320800781, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.310967922210693, "metrics/particle/momentum/class_2/event/px_bias": 0.9447187185287476, "metrics/particle/momentum/class_2/event/py_bias": -0.657017171382904, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.777275085449219, "metrics/particle/momentum/class_2/event/pz_bias": -0.6041097640991211, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.946571350097656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019887704402208328, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14938022196292877, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1507863700389862, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03831097111105919, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1906493902206421, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013513513840734959, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2567811608314514, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.06180478632450104, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9063650965690613, "metrics/particle/momentum/class_3/accepted/samples": 11840.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25136205554008484, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5448122024536133, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4551165103912354, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3237314224243164, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1602127104997635, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.85434341430664, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008249250240623951, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5487765669822693, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02886250801384449, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.64939546585083, "metrics/particle/momentum/class_3/assigned/samples": 14668.0, "metrics/particle/momentum/class_3/efficiency": 0.7785888314247131, "metrics/particle/momentum/class_3/event/energy_bias": -0.5395433902740479, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.910507202148438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.66886043548584, "metrics/particle/momentum/class_3/event/px_bias": -0.2532947063446045, "metrics/particle/momentum/class_3/event/py_bias": -0.768727719783783, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.619157314300537, "metrics/particle/momentum/class_3/event/pz_bias": 0.9097775220870972, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.29794979095459, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023799406364560127, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15684059262275696, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15223857760429382, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04606236517429352, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19271275401115417, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010791366919875145, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7279534935951233, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2696917653083801, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7256953716278076, "metrics/particle/momentum/class_4/accepted/samples": 278.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.32949331402778625, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3973757028579712, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1727246046066284, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.0426316261291504, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1774647831916809, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.246779918670654, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0402408838272095, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2501426935195923, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4984867572784424, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.7637362480163574, "metrics/particle/momentum/class_4/event/energy_bias": -0.7840754985809326, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.439725875854492, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.630717158317566, "metrics/particle/momentum/class_4/event/px_bias": -0.03181121498346329, "metrics/particle/momentum/class_4/event/py_bias": -0.09036439657211304, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5082210302352905, "metrics/particle/momentum/class_4/event/pz_bias": 0.5173821449279785, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.752482891082764, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01684064045548439, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13393452763557434, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13475704193115234, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01355574931949377, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17753861844539642, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03870967775583267, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7734568119049072, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.20227296650409698, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.7992360591888428, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08439116925001144, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.37083521485328674, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.26671671867370605, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10383214801549911, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.01886792480945587, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.6027642488479614, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.044025156646966934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9304600954055786, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.36743614077568054, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.32175874710083, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9748427867889404, "metrics/particle/momentum/class_5/event/energy_bias": -0.6380840539932251, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7910287380218506, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3441659212112427, "metrics/particle/momentum/class_5/event/px_bias": -0.05015558749437332, "metrics/particle/momentum/class_5/event/py_bias": 0.17825432121753693, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9535945653915405, "metrics/particle/momentum/class_5/event/pz_bias": 0.09601952880620956, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.117414712905884, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3349333107471466, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.33669567108154297, "metrics/set_slot/angular_recall_all_dr0p10": 0.9574930667877197, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8503243327140808, "metrics/set_slot/soft_count_bias_mean": -0.4579625129699707}, "step_train_time": 5.154649972915649, "step_valid_time": 8798.60759973526, "step_total_time": 8803.762249946594, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001892661530811, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12511329261331428, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9554013875123885, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579330904272938, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566655640092623, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042066909572706195, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9497852659398744, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523020867837032, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9510420112471055, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9554013875123885, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993901027174052, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11570878012887087, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2145, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2138, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9482758620689655, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953757225433526, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9510086455331412, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046242774566473986, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9451812555260831, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506447309915518, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9479051208157836, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9482758620689655, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9946275870970559, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12188117822050226, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1202, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9443137254901961, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9395239953179867, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05568627450980392, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9332298136645962, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9427450980392157, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9379633242294186, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347826086956522} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_33000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..d5f58b1d626bdcb3602851e3ccddce6718a79ea0 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5633882888004549, "train": {"Classification_binary": 0.12133593112230301, "Cardinality": 0.09804989397525787, "Classification": 0.06176045164465904, "Regression_pt": 0.10686872154474258, "Regression_eta": 0.0011550466297194362, "Regression_sin_phi": 0.0007530249422416091, "Regression_cos_phi": 0.0007246665190905333, "Regression_energy": 0.10518660396337509, "HitGrouping": 0.15137407183647156, "Auxiliary": 0.17010213434696198, "Total": 0.8327847123146057, "Total_unweighted": 0.3977844463661313}, "train_group": {}, "valid": {"Classification_binary": 0.14416280388832092, "Cardinality": 0.1064119537671407, "Classification": 0.07362242539723714, "Regression_pt": 0.1056184967358907, "Regression_eta": 0.0013353143197794755, "Regression_sin_phi": 0.0007953411744286617, "Regression_cos_phi": 0.0008030470926314592, "Regression_energy": 0.10466787219047546, "HitGrouping": 0.14941787719726562, "Auxiliary": 0.18467819690704346, "Total": 0.8968058427174886, "Total_unweighted": 0.4310053007987638, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8427203893661499, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5980815887451172, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2915840148925781, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6038270592689514, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6155514121055603, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3470293879508972, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7938748002052307, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5453239679336548, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3043036162853241, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5522492527961731, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5720688104629517, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4208742678165436, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8542075157165527, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5794752240180969, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2792583405971527, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5830896496772766, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5891510844230652, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32465654611587524, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8450994491577148, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5823993682861328, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.285471111536026, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5869965553283691, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5961143970489502, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3404863178730011, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026579219847917557, "metrics/hit_grouping/calorimeter/background_recall": 0.6482131481170654, "metrics/hit_grouping/calorimeter/completeness": 0.652733325958252, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6630467176437378, "metrics/hit_grouping/calorimeter/energy_purity": 0.6554978489875793, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32254090905189514, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.861458420753479, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5914735794067383, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22848612070083618, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5986153483390808, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6001549959182739, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2911316752433777, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8494377136230469, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6019995212554932, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.29427462816238403, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6075388789176941, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6195086240768433, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.34509947896003723, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8059520125389099, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5427671670913696, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3969156742095947, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.540996253490448, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.563182532787323, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.44014662504196167, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8318541049957275, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6184781789779663, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3337900936603546, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6263764500617981, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6496222615242004, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.38648396730422974, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.824124813079834, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6864545941352844, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.42229199409484863, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6808385848999023, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6970470547676086, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3851386606693268, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8620229959487915, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5302972197532654, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21601714193820953, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5326523184776306, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5256743431091309, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28532207012176514, "metrics/hit_grouping/calorimeter/purity": 0.643641471862793, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8146520256996155, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.15171721577644348, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1246153861284256, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15097253024578094, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.1113058477640152, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8109740614891052, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3061089813709259, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11744584143161774, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3059231638908386, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.30373796820640564, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2360319197177887, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8477111458778381, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6074700951576233, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.29856374859809875, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6124561429023743, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6232638955116272, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3528331518173218, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7426989674568176, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4043634235858917, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5361759066581726, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5764576196670532, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5670950412750244, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8005886673927307, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30678632855415344, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.577072262763977, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5960853695869446, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4139091372489929, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6813346743583679, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4221469759941101, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.45564112067222595, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5024775266647339, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.655272901058197, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7107735276222229, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4068108797073364, "metrics/hit_grouping/tracker/all/object_completeness": 0.49132269620895386, "metrics/hit_grouping/tracker/all/object_purity": 0.5337441563606262, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6070753931999207, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0042396532371640205, "metrics/hit_grouping/tracker/background_recall": 0.04021796956658363, "metrics/hit_grouping/tracker/completeness": 0.5026658177375793, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4319785535335541, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7471638917922974, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3916172981262207, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5688584446907043, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5925300121307373, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5605466961860657, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6870803236961365, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.424709677696228, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45995286107063293, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5181580781936646, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6461069583892822, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.65516597032547, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42152103781700134, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35253435373306274, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.41980791091918945, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6688916087150574, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7091845870018005, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41633716225624084, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49027785658836365, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5497625470161438, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6030732989311218, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.703318178653717, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41944074630737305, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46910449862480164, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5367653369903564, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6171770691871643, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7141544818878174, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39217981696128845, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.49659061431884766, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5123452544212341, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6105127334594727, "metrics/hit_grouping/tracker/purity": 0.47881102561950684, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8921515941619873, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.056907039135694504, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.15903745591640472, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15099547803401947, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.699684739112854, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2892690598964691, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.37434902787208557, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.38337379693984985, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.542249858379364, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6873899698257446, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4677409827709198, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.54979407787323, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6028754115104675, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.68644118309021, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5888573527336121, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7075859904289246, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8027902841567993, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6548412442207336, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10313001275062561, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12403646856546402, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.813013076782227, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3178108334541321, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18737101554870605, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9058963060379028, "metrics/particle/class_1/pid_accuracy": 0.9738991856575012, "metrics/particle/class_2/efficiency": 0.5390397310256958, "metrics/particle/class_2/pid_accuracy": 0.7660769820213318, "metrics/particle/class_3/efficiency": 0.7923969626426697, "metrics/particle/class_3/pid_accuracy": 0.9771226048469543, "metrics/particle/class_4/efficiency": 0.7555341720581055, "metrics/particle/class_4/pid_accuracy": 0.5541401505470276, "metrics/particle/class_5/efficiency": 0.9279999732971191, "metrics/particle/class_5/pid_accuracy": 0.6293103694915771, "metrics/particle/count/bias_mean": -1.4967447519302368, "metrics/particle/count/mae": 3.2272136211395264, "metrics/particle/count/prediction_mean": 68.55924224853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05895628407597542, "metrics/particle/event/energy_response_mean": 0.9767475128173828, "metrics/particle/event/met_abs_error": 6.146825313568115, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04736548662185669, "metrics/particle/event/scalar_pt_response_mean": 0.9926057457923889, "metrics/particle/event/vector_pt_closure": 0.05008256435394287, "metrics/particle/matched/delta_eta_abs_mean": 0.013344261795282364, "metrics/particle/matched/delta_phi_abs_mean": 0.01457842718809843, "metrics/particle/matched/delta_r_mean": 0.02218446135520935, "metrics/particle/matched/energy_relative_abs_mean": 0.1299164891242981, "metrics/particle/matched/pid_accuracy": 0.9584788084030151, "metrics/particle/matched/pt_relative_abs_mean": 0.13116604089736938, "metrics/particle/matching/duplicate_fraction": 0.04741375148296356, "metrics/particle/matching/efficiency": 0.8249911665916443, "metrics/particle/matching/f1": 0.8338993191719055, "metrics/particle/matching/purity": 0.8430019021034241, "metrics/particle/matching/target_coverage_dr0p05": 0.7762020826339722, "metrics/particle/matching/target_coverage_dr0p10": 0.8671170473098755, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6549263000488281, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8578982353210449, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022055653855204582, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12819869816303253, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1295350342988968, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008205627091228962, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16851502656936646, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006027458235621452, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6351156234741211, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008170186541974545, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7707995176315308, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8959.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21213041245937347, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7511181831359863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7915388941764832, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6382532119750977, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1066296249628067, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.6526618003845215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012262885458767414, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9050428867340088, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.004853460006415844, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.785154342651367, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10438.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8192209005355835, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.6614084243774414, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.550256729125977, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3681347370147705, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.16152960062026978, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.33435314893722534, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.491703987121582, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5757570862770081, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.890554428100586, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02228773757815361, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1372513771057129, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13594232499599457, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.000462274911114946, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17359548807144165, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025641025975346565, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3472003936767578, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.053610097616910934, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.55792236328125, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2067.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.41055864095687866, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9523003697395325, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.972123622894287, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8019483089447021, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1592210829257965, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 16.828508377075195, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.054219167679548264, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.266563653945923, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05130702257156372, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.955689430236816, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2619.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6842105388641357, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.10936164855957, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.686918258666992, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.212309718132019, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.021475717425346375, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.17358948290348053, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.51518440246582, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.17086052894592285, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.345439910888672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021876344457268715, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12948985397815704, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13057240843772888, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024544870480895042, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17033836245536804, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006079027079977095, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21271051466464996, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004110449459403753, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6864027380943298, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18095.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16887998580932617, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9520513415336609, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7264133095741272, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5875244140625, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10939358174800873, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.147024154663086, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003281807294115424, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3553818464279175, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0022743844892829657, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4027540683746338, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21025.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8441013097763062, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.33781224489212036, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.7889299392700195, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.682883262634277, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7933311462402344, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3672804534435272, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.336482048034668, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.08008274435997009, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.664433479309082, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021960707381367683, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12964355945587158, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13063441216945648, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017743144184350967, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17001350224018097, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004052058793604374, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.42318814992904663, "metrics/particle/momentum/all/accepted/pz_bias": -0.00887288898229599, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4683246612548828, "metrics/particle/momentum/all/accepted/samples": 29121.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20069749653339386, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.890532374382019, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8420842289924622, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6963819861412048, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11237603425979614, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.174813270568848, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00994659960269928, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6705849170684814, "metrics/particle/momentum/all/assigned/pz_bias": -0.001053158426657319, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5130159854888916, "metrics/particle/momentum/all/assigned/samples": 34082.0, "metrics/particle/momentum/all/efficiency": 0.8227665424346924, "metrics/particle/momentum/all/event/energy_bias": -8.108582496643066, "metrics/particle/momentum/all/event/p3_error_mean": 16.771690368652344, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.427933692932129, "metrics/particle/momentum/all/event/px_bias": 0.9763365387916565, "metrics/particle/momentum/all/event/py_bias": -0.20651677250862122, "metrics/particle/momentum/all/event/pz_abs_mean": 12.392467498779297, "metrics/particle/momentum/all/event/pz_bias": -0.4849792718887329, "metrics/particle/momentum/all/event/pz_residual_rms": 20.371803283691406, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023809507489204407, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10908573120832443, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10932830721139908, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0009291573660448194, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14409348368644714, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0038331900723278522, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.43262696266174316, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0022702827118337154, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4333200454711914, "metrics/particle/momentum/class_1/accepted/samples": 15131.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10427603125572205, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2476196438074112, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.25680088996887207, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12247557193040848, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05107707157731056, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.1892635822296143, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006590589880943298, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5511565208435059, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.023620132356882095, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9150540828704834, "metrics/particle/momentum/class_1/assigned/samples": 16387.0, "metrics/particle/momentum/class_1/efficiency": 0.9067537784576416, "metrics/particle/momentum/class_1/event/energy_bias": 0.5221312046051025, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.400635719299316, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.470338821411133, "metrics/particle/momentum/class_1/event/px_bias": 0.8234971761703491, "metrics/particle/momentum/class_1/event/py_bias": -0.032030947506427765, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.988104820251465, "metrics/particle/momentum/class_1/event/pz_bias": 1.4143450260162354, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.282751083374023, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02867473103106022, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17728126049041748, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17883305251598358, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.01746026612818241, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22000718116760254, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021464645862579346, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4009215831756592, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.07302305847406387, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4345109462738037, "metrics/particle/momentum/class_2/accepted/samples": 1584.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5610544085502625, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.603190004825592, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8628054857254028, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5479670166969299, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.263200968503952, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.507047176361084, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04461727291345596, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1875901222229004, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.021452181041240692, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.354809284210205, "metrics/particle/momentum/class_2/assigned/samples": 2443.0, "metrics/particle/momentum/class_2/efficiency": 0.53207927942276, "metrics/particle/momentum/class_2/event/energy_bias": -10.409173965454102, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.39401912689209, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.24593448638916, "metrics/particle/momentum/class_2/event/px_bias": 0.4319893717765808, "metrics/particle/momentum/class_2/event/py_bias": -0.824407160282135, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.727299690246582, "metrics/particle/momentum/class_2/event/pz_bias": -0.9862207770347595, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.422645568847656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01875746250152588, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14866827428340912, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15068772435188293, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03875439986586571, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19073091447353363, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015851827338337898, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2571452856063843, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01010130438953638, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9252392053604126, "metrics/particle/momentum/class_3/accepted/samples": 11986.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24658431112766266, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.633312702178955, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4762852191925049, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3456344604492188, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15474897623062134, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.011622428894043, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007530529052019119, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5293264985084534, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.02129119075834751, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3034188747406006, "metrics/particle/momentum/class_3/assigned/samples": 14740.0, "metrics/particle/momentum/class_3/efficiency": 0.7881896495819092, "metrics/particle/momentum/class_3/event/energy_bias": 2.888838052749634, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.992667198181152, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.815007209777832, "metrics/particle/momentum/class_3/event/px_bias": -0.05824629217386246, "metrics/particle/momentum/class_3/event/py_bias": 0.2647956311702728, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.545985221862793, "metrics/particle/momentum/class_3/event/pz_bias": -0.9721413850784302, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.057421684265137, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025364341214299202, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14947739243507385, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14435328543186188, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03802391514182091, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19007360935211182, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007326007355004549, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8140175342559814, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1945430189371109, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9433655738830566, "metrics/particle/momentum/class_4/accepted/samples": 273.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.31446871161460876, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.0273277759552, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6830790042877197, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.563299298286438, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1614730805158615, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.030783653259277, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0557688474655151, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05283621698617935, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6358883380889893, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.75, "metrics/particle/momentum/class_4/event/energy_bias": -0.6931170225143433, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5250003337860107, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6438539028167725, "metrics/particle/momentum/class_4/event/px_bias": -0.07135970890522003, "metrics/particle/momentum/class_4/event/py_bias": -0.007498875260353088, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6392037868499756, "metrics/particle/momentum/class_4/event/pz_bias": 0.3387959599494934, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.44077205657959, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014176390133798122, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1443176418542862, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14377620816230774, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0006205885438248515, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1844843029975891, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03401360660791397, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7289304733276367, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.7422214150428772, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4230921268463135, "metrics/particle/momentum/class_5/accepted/samples": 147.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09487248212099075, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1831267774105072, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1843750774860382, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01196946483105421, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07547169923782349, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.26125243306159973, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03773584961891174, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9108682870864868, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.8986237049102783, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7319140434265137, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9245283007621765, "metrics/particle/momentum/class_5/event/energy_bias": -0.41726112365722656, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9122155904769897, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4709534645080566, "metrics/particle/momentum/class_5/event/px_bias": -0.14954400062561035, "metrics/particle/momentum/class_5/event/py_bias": 0.3926246762275696, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9779452681541443, "metrics/particle/momentum/class_5/event/pz_bias": -0.27975791692733765, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2559142112731934, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.32755333185195923, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3301059305667877, "metrics/set_slot/angular_recall_all_dr0p10": 0.9583109021186829, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8513744473457336, "metrics/set_slot/soft_count_bias_mean": -0.2836695611476898}, "step_train_time": 4.877872705459595, "step_valid_time": 11810.237035274506, "step_total_time": 11815.114908456802, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019175408895469, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12124260762107364, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9593657086223984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580996370834708, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9587322548695939, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041900362916529196, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520977865873802, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9508413064995052, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9514691317266425, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9593657086223984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9893435330205401, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12245357528394207, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2153, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9518125552608311, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568888888888889, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9543439716312057, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043111111111111114, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9478337754199824, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9528888888888889, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.950354609929078, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9518125552608311, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9967704767692283, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11835225318796772, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1211, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9433229813664596, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9433229813664596, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9433229813664596, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056677018633540376, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9402173913043478, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9402173913043478, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9402173913043478, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9433229813664596} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_34000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..b864bc13a83b8d968c22619a2f6eba6954c9011c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.6107636914913777, "train": {"Classification_binary": 0.12430017441511154, "Cardinality": 0.10365556925535202, "Classification": 0.0641632229089737, "Regression_pt": 0.10470829159021378, "Regression_eta": 0.0013133655302226543, "Regression_sin_phi": 0.0007580620003864169, "Regression_cos_phi": 0.0006778321112506092, "Regression_energy": 0.10326766222715378, "HitGrouping": 0.1517699956893921, "Auxiliary": 0.17063751816749573, "Total": 0.8428628444671631, "Total_unweighted": 0.39918861078331247}, "train_group": {}, "valid": {"Classification_binary": 0.14175613721211752, "Cardinality": 0.11419542630513509, "Classification": 0.07047111789385478, "Regression_pt": 0.09999621907869975, "Regression_eta": 0.0013224758828679721, "Regression_sin_phi": 0.0007670684717595577, "Regression_cos_phi": 0.0007799048908054829, "Regression_energy": 0.0996427337328593, "HitGrouping": 0.14781532684961954, "Auxiliary": 0.18384728829065958, "Total": 0.8880434036254883, "Total_unweighted": 0.41473565716296434, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8477000594139099, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6028676629066467, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28780579566955566, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6085218787193298, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6152825355529785, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33626145124435425, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7928800582885742, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5491235256195068, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30870890617370605, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5553306341171265, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5740720629692078, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.42799052596092224, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8580565452575684, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5823724865913391, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2732672393321991, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5861865878105164, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5897542238235474, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31778740882873535, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8488777875900269, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5859854221343994, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.28112462162971497, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5906062722206116, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5965624451637268, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33361703157424927, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026667369529604912, "metrics/hit_grouping/calorimeter/background_recall": 0.6499959826469421, "metrics/hit_grouping/calorimeter/completeness": 0.6616573333740234, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6726775765419006, "metrics/hit_grouping/calorimeter/energy_purity": 0.6650727391242981, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32007360458374023, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8647863864898682, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5925986170768738, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2234099805355072, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5999735593795776, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5970977544784546, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28241264820098877, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8529423475265503, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6112215518951416, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.28966522216796875, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6155688166618347, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6252123117446899, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.339277058839798, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8109690546989441, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5456889867782593, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3943875730037689, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5447655916213989, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5655733346939087, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43610164523124695, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8376656770706177, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6284308433532715, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3316100835800171, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6360914707183838, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6557270884513855, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3751479387283325, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8308765888214111, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.704563319683075, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4207221269607544, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7010524272918701, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.711645245552063, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38042908906936646, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8632350564002991, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5251803398132324, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20898620784282684, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.527573823928833, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5182303190231323, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28297844529151917, "metrics/hit_grouping/calorimeter/purity": 0.6525002717971802, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8144523501396179, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.16756105422973633, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11692307889461517, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1655702143907547, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12583550810813904, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8029050827026367, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30069470405578613, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1145952120423317, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.29945626854896545, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28523361682891846, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24686430394649506, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8522557020187378, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6113125085830688, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2942025065422058, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6164112687110901, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6245019435882568, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3448649048805237, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7426586151123047, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4125821888446808, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.54112309217453, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5856346487998962, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.565002977848053, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7976725101470947, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3107122778892517, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5739511251449585, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5909936428070068, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42680874466896057, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6769722700119019, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43136173486709595, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4629359245300293, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5134533047676086, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6592108607292175, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7078989744186401, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41526544094085693, "metrics/hit_grouping/tracker/all/object_completeness": 0.4969983696937561, "metrics/hit_grouping/tracker/all/object_purity": 0.5427982807159424, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6098620891571045, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004212202038615942, "metrics/hit_grouping/tracker/background_recall": 0.03536101430654526, "metrics/hit_grouping/tracker/completeness": 0.5117019414901733, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4303341805934906, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7421573400497437, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3952619731426239, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5686518549919128, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6014485955238342, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5662870407104492, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.687588095664978, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4344184398651123, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.46743541955947876, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.525233268737793, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6426803469657898, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6534305214881897, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4393203854560852, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.36934801936149597, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.43126553297042847, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6717233061790466, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7095417380332947, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42316678166389465, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.497364342212677, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5539694428443909, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6023544073104858, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6943698525428772, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45073235034942627, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48160016536712646, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5443016290664673, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6304926872253418, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7081379294395447, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3987356722354889, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4992291033267975, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5279807448387146, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6160149574279785, "metrics/hit_grouping/tracker/purity": 0.48740488290786743, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8938615918159485, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05951346829533577, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.15822334587574005, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15412762761116028, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7004340887069702, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2882322371006012, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.39216193556785583, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4095657467842102, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5230689644813538, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6834797859191895, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4780709445476532, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5549695491790771, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6107827425003052, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6921079158782959, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5945000648498535, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7281762957572937, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8269296288490295, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6619303226470947, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10877276211977005, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1311255246400833, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.67534637451172, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29858991503715515, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1681501269340515, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9077166318893433, "metrics/particle/class_1/pid_accuracy": 0.9742131233215332, "metrics/particle/class_2/efficiency": 0.5579986572265625, "metrics/particle/class_2/pid_accuracy": 0.7878970503807068, "metrics/particle/class_3/efficiency": 0.7955291271209717, "metrics/particle/class_3/pid_accuracy": 0.9783183336257935, "metrics/particle/class_4/efficiency": 0.7478344440460205, "metrics/particle/class_4/pid_accuracy": 0.5868725776672363, "metrics/particle/class_5/efficiency": 0.949999988079071, "metrics/particle/class_5/pid_accuracy": 0.6547368168830872, "metrics/particle/count/bias_mean": -2.3430988788604736, "metrics/particle/count/mae": 3.5436198711395264, "metrics/particle/count/prediction_mean": 67.712890625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05744621530175209, "metrics/particle/event/energy_response_mean": 0.9880914092063904, "metrics/particle/event/met_abs_error": 5.883771896362305, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045646075159311295, "metrics/particle/event/scalar_pt_response_mean": 1.0002388954162598, "metrics/particle/event/vector_pt_closure": 0.04712904989719391, "metrics/particle/matched/delta_eta_abs_mean": 0.013459793291985989, "metrics/particle/matched/delta_phi_abs_mean": 0.014450866729021072, "metrics/particle/matched/delta_r_mean": 0.022148894146084785, "metrics/particle/matched/energy_relative_abs_mean": 0.12937214970588684, "metrics/particle/matched/pid_accuracy": 0.9604525566101074, "metrics/particle/matched/pt_relative_abs_mean": 0.12981745600700378, "metrics/particle/matching/duplicate_fraction": 0.04418933391571045, "metrics/particle/matching/efficiency": 0.8288013935089111, "metrics/particle/matching/f1": 0.8428971767425537, "metrics/particle/matching/purity": 0.8574807643890381, "metrics/particle/matching/target_coverage_dr0p05": 0.7778655290603638, "metrics/particle/matching/target_coverage_dr0p10": 0.864477813243866, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6602698564529419, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8600821495056152, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021734286099672318, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12492257356643677, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12604278326034546, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020423641428351402, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16557438671588898, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0065606581047177315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6177720427513123, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.014772851020097733, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.721944808959961, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8993.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19273076951503754, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8038620352745056, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8125502467155457, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6839669942855835, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10402845591306686, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.185449600219727, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013460244052112103, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8767510652542114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.011283127591013908, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.592252731323242, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10401.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8223299384117126, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5576677322387695, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.713688373565674, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.068988084793091, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2969711422920227, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1360282301902771, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.722851753234863, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.03577572852373123, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.158905982971191, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022083278745412827, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13333719968795776, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13334108889102936, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011861767619848251, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1719353348016739, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023222748190164566, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3542420864105225, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.07764323800802231, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.073497295379639, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2110.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3347584903240204, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8205482959747314, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9649794101715088, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8086895942687988, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.13466717302799225, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.885557174682617, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05309734493494034, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1939244270324707, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07218995690345764, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.069339275360107, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2599.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6984442472457886, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.335724115371704, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.4690351486206055, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0653877258300781, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06279728561639786, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05082585662603378, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.334606170654297, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3690404891967773, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.708675384521484, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022206569090485573, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.130271777510643, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1301020383834839, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02503550797700882, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1697724461555481, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004940711660310626, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21380671858787537, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011289788410067558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6894117593765259, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18216.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1713976263999939, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0172611474990845, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7834209203720093, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6472784876823425, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09860233217477798, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.304920196533203, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003542025573551655, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.36912745237350464, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0034053947310894728, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7347297668457031, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20892.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8497457504272461, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.8366910219192505, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.523204803466797, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.361797332763672, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8494950532913208, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.532625675201416, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.3449015617370605, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4223726689815521, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.174597263336182, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022052833810448647, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12885163724422455, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1290900558233261, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02267283946275711, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.168653666973114, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003990586381405592, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4197884798049927, "metrics/particle/momentum/all/accepted/pz_bias": -0.017133407294750214, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.548856258392334, "metrics/particle/momentum/all/accepted/samples": 29319.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19047176837921143, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9366869330406189, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8829678893089294, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.747600257396698, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1030331626534462, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.590519905090332, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010385931469500065, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6648445129394531, "metrics/particle/momentum/all/assigned/pz_bias": -0.004172412678599358, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5965449810028076, "metrics/particle/momentum/all/assigned/samples": 33892.0, "metrics/particle/momentum/all/efficiency": 0.8283607363700867, "metrics/particle/momentum/all/event/energy_bias": -2.0567009449005127, "metrics/particle/momentum/all/event/p3_error_mean": 15.997790336608887, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.717845439910889, "metrics/particle/momentum/all/event/px_bias": 1.0836689472198486, "metrics/particle/momentum/all/event/py_bias": -0.4474233388900757, "metrics/particle/momentum/all/event/pz_abs_mean": 12.028663635253906, "metrics/particle/momentum/all/event/pz_bias": -0.9824435710906982, "metrics/particle/momentum/all/event/pz_residual_rms": 19.549623489379883, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0235744696110487, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10760848969221115, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10787650942802429, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03326711058616638, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14372099936008453, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0034264628775417805, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.41596323251724243, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0055775889195501804, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.47516930103302, "metrics/particle/momentum/class_1/accepted/samples": 15176.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10098253935575485, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32430219650268555, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.29106882214546204, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1950160562992096, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.050933267921209335, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.733738422393799, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006709771696478128, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5499782562255859, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005211217328906059, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.224730968475342, "metrics/particle/momentum/class_1/assigned/samples": 16394.0, "metrics/particle/momentum/class_1/efficiency": 0.9094504714012146, "metrics/particle/momentum/class_1/event/energy_bias": 8.209455490112305, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.823131561279297, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.834075927734375, "metrics/particle/momentum/class_1/event/px_bias": 1.5305290222167969, "metrics/particle/momentum/class_1/event/py_bias": -0.6479788422584534, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.749150276184082, "metrics/particle/momentum/class_1/event/pz_bias": 0.15269872546195984, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.780733108520508, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02757464535534382, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1804746389389038, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18108314275741577, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.014438415877521038, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22180922329425812, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026780279353260994, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5009651184082031, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1296582818031311, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8515682220458984, "metrics/particle/momentum/class_2/accepted/samples": 1643.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4927271008491516, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.681697428226471, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8034051656723022, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.48550817370414734, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.22511549293994904, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.838896751403809, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05081898346543312, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2849068641662598, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.014277181588113308, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.793888092041016, "metrics/particle/momentum/class_2/assigned/samples": 2381.0, "metrics/particle/momentum/class_2/efficiency": 0.5518978834152222, "metrics/particle/momentum/class_2/event/energy_bias": -9.99641227722168, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.119050025939941, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.759696960449219, "metrics/particle/momentum/class_2/event/px_bias": -0.031879447400569916, "metrics/particle/momentum/class_2/event/py_bias": -0.23829345405101776, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.691577911376953, "metrics/particle/momentum/class_2/event/pz_bias": -1.3900160789489746, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.571718215942383, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01937071792781353, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1479203850030899, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14826619625091553, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013630776666104794, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18769319355487823, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012421332066878676, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.25202012062072754, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01801876164972782, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8997167944908142, "metrics/particle/momentum/class_3/accepted/samples": 12076.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23994559049606323, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6434717178344727, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5440335273742676, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3937257528305054, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1404121369123459, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.638901710510254, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007599096279591322, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5064473748207092, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.027472207322716713, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.0382649898529053, "metrics/particle/momentum/class_3/assigned/samples": 14607.0, "metrics/particle/momentum/class_3/efficiency": 0.794107973575592, "metrics/particle/momentum/class_3/event/energy_bias": 0.4672326147556305, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.190084457397461, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.2672200202941895, "metrics/particle/momentum/class_3/event/px_bias": -0.25347161293029785, "metrics/particle/momentum/class_3/event/py_bias": 0.14069333672523499, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.019599914550781, "metrics/particle/momentum/class_3/event/pz_bias": 0.21752861142158508, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.983729362487793, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026124417781829834, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15292046964168549, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14658324420452118, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.054779667407274246, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.192142516374588, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007326007355004549, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7190554141998291, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17969244718551636, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6725047826766968, "metrics/particle/momentum/class_4/accepted/samples": 273.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3136250674724579, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.101228952407837, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8234630823135376, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7087185382843018, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.17897726595401764, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.366470336914062, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014204545877873898, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0082627534866333, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.42053911089897156, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.174126386642456, "metrics/particle/momentum/class_4/assigned/samples": 352.0, "metrics/particle/momentum/class_4/efficiency": 0.75, "metrics/particle/momentum/class_4/event/energy_bias": -0.5281842350959778, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1669154167175293, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5415668487548828, "metrics/particle/momentum/class_4/event/px_bias": -0.08664888143539429, "metrics/particle/momentum/class_4/event/py_bias": 0.1432802975177765, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2816545963287354, "metrics/particle/momentum/class_4/event/pz_bias": 0.04080502688884735, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8337087631225586, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01617857813835144, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13364830613136292, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13018347322940826, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02679113671183586, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16567236185073853, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026490066200494766, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9161380529403687, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.23921650648117065, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9849355220794678, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07276179641485214, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3837703764438629, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2867087721824646, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1579689234495163, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04430379718542099, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.6954474449157715, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03164556995034218, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.0482139587402344, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04798305034637451, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.50093936920166, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9496855139732361, "metrics/particle/momentum/class_5/event/energy_bias": -0.20879170298576355, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7863291501998901, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3130677938461304, "metrics/particle/momentum/class_5/event/px_bias": -0.07486024498939514, "metrics/particle/momentum/class_5/event/py_bias": 0.15487530827522278, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9898710250854492, "metrics/particle/momentum/class_5/event/pz_bias": -0.003459801897406578, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.064857006072998, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.32254284620285034, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.32406166195869446, "metrics/set_slot/angular_recall_all_dr0p10": 0.9589613676071167, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8500919938087463, "metrics/set_slot/soft_count_bias_mean": -1.2265739440917969}, "step_train_time": 4.859525442123413, "step_valid_time": 14824.875606298447, "step_total_time": 14829.735132217407, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013858948815009, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12145305884049062, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2905, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2890, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9596960687148992, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9587458745874587, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9592207363381212, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041254125412541254, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547406673273868, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9537953795379538, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9542677893346542, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9596960687148992, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00553135774821, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.115491189193624, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9531388152077808, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9616413916146298, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573712255772646, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0383586083853702, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496021220159151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9580731489741302, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9538188277087034, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9531388152077808, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988982133447721, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12191411617232617, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1202, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9332298136645962, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9346811819595645, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.933954933954934, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06531881804043546, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9316770186335404, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9331259720062208, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9324009324009324, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9332298136645962} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_35000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..1ca5ed97d2936ba09c84e073d7783ae814bb65ee --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6581390941823007, "train": {"Classification_binary": 0.13550686836242676, "Cardinality": 0.11883049458265305, "Classification": 0.0645638108253479, "Regression_pt": 0.09880749881267548, "Regression_eta": 0.0013415539870038629, "Regression_sin_phi": 0.000792471575550735, "Regression_cos_phi": 0.0007907539256848395, "Regression_energy": 0.0986098200082779, "HitGrouping": 0.14987044036388397, "Auxiliary": 0.1813960075378418, "Total": 0.8845967650413513, "Total_unweighted": 0.40041277749696746}, "train_group": {}, "valid": {"Classification_binary": 0.13982575138409933, "Cardinality": 0.10282540321350098, "Classification": 0.06781844794750214, "Regression_pt": 0.09895987311999004, "Regression_eta": 0.0013155220076441765, "Regression_sin_phi": 0.0007454801816493273, "Regression_cos_phi": 0.0007451614364981651, "Regression_energy": 0.0980146328608195, "HitGrouping": 0.14703591664632162, "Auxiliary": 0.18122675021489462, "Total": 0.8639565308888754, "Total_unweighted": 0.4074248689382027, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8479344248771667, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.600717306137085, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2778879702091217, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6065143346786499, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6161403656005859, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3404175043106079, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7932766675949097, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.554847776889801, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3002372086048126, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.561802327632904, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5808806419372559, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4259572923183441, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8565329909324646, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5790396928787231, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27233755588531494, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5827223658561707, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5903464555740356, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3208862841129303, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8480921983718872, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5838454961776733, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2766261398792267, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5885019898414612, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5977977514266968, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.336595743894577, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026323191821575165, "metrics/hit_grouping/calorimeter/background_recall": 0.6450114250183105, "metrics/hit_grouping/calorimeter/completeness": 0.6594666838645935, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6693753600120544, "metrics/hit_grouping/calorimeter/energy_purity": 0.6616237759590149, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3124041259288788, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8661607503890991, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.590840220451355, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22149895131587982, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5983707308769226, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5972729921340942, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28587639331817627, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8498464822769165, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6094288229942322, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2860262095928192, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6139494180679321, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6286971569061279, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3387918472290039, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8080189824104309, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5423795580863953, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3835166096687317, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5410944819450378, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5667082667350769, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4416635036468506, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8341410756111145, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6267043948173523, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32376205921173096, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6338867545127869, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6589563488960266, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3814387917518616, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8329366445541382, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7000322341918945, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.41810569167137146, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6987063884735107, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7046433091163635, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.36734694242477417, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8650299310684204, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5229017734527588, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20784786343574524, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5256084203720093, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5183759927749634, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28445157408714294, "metrics/hit_grouping/calorimeter/purity": 0.650110125541687, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8160988092422485, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.13743750751018524, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1246153861284256, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.13418680429458618, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0958595871925354, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03999999910593033, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8211448192596436, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2856068015098572, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10718358308076859, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2855744957923889, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2824651598930359, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24743443727493286, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8503239750862122, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6105137467384338, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2896117568016052, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6156085133552551, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6266326904296875, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3480456471443176, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7477980852127075, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3876270055770874, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5415050983428955, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5935243964195251, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.559473991394043, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7965818643569946, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.29949522018432617, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5825507044792175, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6150863766670227, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4318564236164093, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6859971880912781, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4236433804035187, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.46538424491882324, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5166216492652893, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.64897221326828, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7148439884185791, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40180426836013794, "metrics/hit_grouping/tracker/all/object_completeness": 0.49931153655052185, "metrics/hit_grouping/tracker/all/object_purity": 0.5492211580276489, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6023994088172913, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005334642715752125, "metrics/hit_grouping/tracker/background_recall": 0.051057275384664536, "metrics/hit_grouping/tracker/completeness": 0.5135927796363831, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4214765131473541, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7508337497711182, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.378314346075058, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5742200613021851, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6101656556129456, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5566287040710449, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6931946873664856, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41785645484924316, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4690825045108795, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5350924134254456, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6398248672485352, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6579539179801941, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43689319491386414, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.36514317989349365, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.42894434928894043, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6642394661903381, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.712486743927002, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4041157364845276, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4968767762184143, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.565478503704071, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5973220467567444, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7024310827255249, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4553928077220917, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.47833460569381714, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5368556380271912, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6205059885978699, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7200974225997925, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3893701732158661, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5061719417572021, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5302162766456604, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6058300137519836, "metrics/hit_grouping/tracker/purity": 0.4897579848766327, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.889610230922699, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05734144151210785, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1505430042743683, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15157662332057953, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6937400698661804, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27579057216644287, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.390471875667572, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4271451532840729, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5484707355499268, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6934991478919983, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4629596769809723, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5590965151786804, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6171546578407288, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6798890233039856, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5929185152053833, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7239950299263, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8240444660186768, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6552290916442871, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10719119012355804, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12442431598901749, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.67153549194336, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30210232734680176, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17166249454021454, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9141274094581604, "metrics/particle/class_1/pid_accuracy": 0.9740476012229919, "metrics/particle/class_2/efficiency": 0.5640565156936646, "metrics/particle/class_2/pid_accuracy": 0.7836117744445801, "metrics/particle/class_3/efficiency": 0.8184626698493958, "metrics/particle/class_3/pid_accuracy": 0.9782443642616272, "metrics/particle/class_4/efficiency": 0.7747834324836731, "metrics/particle/class_4/pid_accuracy": 0.6012422442436218, "metrics/particle/class_5/efficiency": 0.9480000138282776, "metrics/particle/class_5/pid_accuracy": 0.6751055121421814, "metrics/particle/count/bias_mean": -0.7766926884651184, "metrics/particle/count/mae": 3.1217448711395264, "metrics/particle/count/prediction_mean": 69.279296875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05775122717022896, "metrics/particle/event/energy_response_mean": 0.9866743087768555, "metrics/particle/event/met_abs_error": 6.004587650299072, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04598579183220863, "metrics/particle/event/scalar_pt_response_mean": 0.9975710511207581, "metrics/particle/event/vector_pt_closure": 0.048744410276412964, "metrics/particle/matched/delta_eta_abs_mean": 0.013780479319393635, "metrics/particle/matched/delta_phi_abs_mean": 0.014050433412194252, "metrics/particle/matched/delta_r_mean": 0.02215813845396042, "metrics/particle/matched/energy_relative_abs_mean": 0.126524418592453, "metrics/particle/matched/pid_accuracy": 0.9603785872459412, "metrics/particle/matched/pt_relative_abs_mean": 0.12730515003204346, "metrics/particle/matching/duplicate_fraction": 0.0454549714922905, "metrics/particle/matching/efficiency": 0.8424994945526123, "metrics/particle/matching/f1": 0.8471958041191101, "metrics/particle/matching/purity": 0.8519448041915894, "metrics/particle/matching/target_coverage_dr0p05": 0.7887850403785706, "metrics/particle/matching/target_coverage_dr0p10": 0.8777205944061279, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6780291199684143, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8740962147712708, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021857459098100662, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12575054168701172, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12669768929481506, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008322778157889843, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16549614071846008, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0057602436281740665, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6112205982208252, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04240095242857933, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7348864078521729, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9201.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1985870599746704, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8310223817825317, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8559547662734985, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7099617719650269, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09923447668552399, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.69589900970459, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012286173179745674, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8915404677391052, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07485207915306091, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9072225093841553, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10581.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8413496613502502, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7233678102493286, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.316781520843506, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.24170184135437, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5866622924804688, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6867647767066956, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.280642032623291, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.24752525985240936, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.32880687713623, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021782681345939636, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12908166646957397, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13034546375274658, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010147958993911743, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16926182806491852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024525683373212814, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.269344687461853, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.058423370122909546, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.361943006515503, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2161.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.404660701751709, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7424036264419556, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.6614943742752075, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.502879023551941, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.15311004221439362, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.271188735961914, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0541037917137146, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2552528381347656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.028110992163419724, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.258955001831055, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2717.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7153260707855225, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2189502716064453, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.292802810668945, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2524313926696777, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05496792122721672, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.022899292409420013, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.154751777648926, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2408527433872223, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.765689849853516, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02213958092033863, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12580783665180206, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12628613412380219, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012857096269726753, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16541045904159546, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006506533827632666, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20861336588859558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.002264112001284957, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6648572087287903, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18443.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1673942357301712, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9408956170082092, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7492460012435913, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.602713406085968, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09636613726615906, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.787903785705566, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034585683606565, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3534218668937683, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012035522609949112, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5386378765106201, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21107.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8603349328041077, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.24105338752269745, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.790054798126221, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.731298923492432, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7954304218292236, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.5465905666351318, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.209637403488159, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.031951744109392166, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.857245445251465, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022026613354682922, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12602750957012177, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12670749425888062, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011260897852480412, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16571912169456482, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003959067165851593, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.40980881452560425, "metrics/particle/momentum/all/accepted/pz_bias": 0.01592441089451313, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.422032356262207, "metrics/particle/momentum/all/accepted/samples": 29805.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19572456181049347, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8914298415184021, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8541046977043152, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7067838907241821, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10172940045595169, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.288296699523926, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010172939859330654, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6691058278083801, "metrics/particle/momentum/all/assigned/pz_bias": 0.013416611589491367, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6730802059173584, "metrics/particle/momentum/all/assigned/samples": 34405.0, "metrics/particle/momentum/all/efficiency": 0.8420918583869934, "metrics/particle/momentum/all/event/energy_bias": -2.7012643814086914, "metrics/particle/momentum/all/event/p3_error_mean": 16.991600036621094, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.51359748840332, "metrics/particle/momentum/all/event/px_bias": 1.3271249532699585, "metrics/particle/momentum/all/event/py_bias": 2.2562546730041504, "metrics/particle/momentum/all/event/pz_abs_mean": 12.405291557312012, "metrics/particle/momentum/all/event/pz_bias": 0.4564262330532074, "metrics/particle/momentum/all/event/pz_residual_rms": 21.24418830871582, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023318538442254066, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10335187613964081, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10475432127714157, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01396891288459301, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14018619060516357, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0037888686638325453, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4189971685409546, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.020342763513326645, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4214072227478027, "metrics/particle/momentum/class_1/accepted/samples": 15308.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10460024327039719, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2552050054073334, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2734142243862152, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15663714706897736, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.046645637601614, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.562391519546509, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0067329853773117065, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5416767001152039, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02592938393354416, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9600743055343628, "metrics/particle/momentum/class_1/assigned/samples": 16486.0, "metrics/particle/momentum/class_1/efficiency": 0.9173608422279358, "metrics/particle/momentum/class_1/event/energy_bias": 4.018285751342773, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.747272491455078, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.411783218383789, "metrics/particle/momentum/class_1/event/px_bias": 0.8649870753288269, "metrics/particle/momentum/class_1/event/py_bias": 1.3352185487747192, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.320779800415039, "metrics/particle/momentum/class_1/event/pz_bias": 0.6948047876358032, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.75544548034668, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02852519601583481, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1744622141122818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17544366419315338, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.004643317777663469, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21430841088294983, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021686747670173645, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3737480640411377, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03309163451194763, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.322155237197876, "metrics/particle/momentum/class_2/accepted/samples": 1660.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5805445909500122, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6783663034439087, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8371151089668274, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5078282356262207, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2549173831939697, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.126441478729248, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04799370467662811, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.346670389175415, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.03767624497413635, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.188855171203613, "metrics/particle/momentum/class_2/assigned/samples": 2542.0, "metrics/particle/momentum/class_2/efficiency": 0.5576083064079285, "metrics/particle/momentum/class_2/event/energy_bias": -9.703786849975586, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.456615447998047, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.738742351531982, "metrics/particle/momentum/class_2/event/px_bias": 0.40077292919158936, "metrics/particle/momentum/class_2/event/py_bias": 0.6954218745231628, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.035338401794434, "metrics/particle/momentum/class_2/event/pz_bias": 0.3843175172805786, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.5565128326416, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01958950236439705, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14690031111240387, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1468210071325302, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.00888911820948124, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18554650247097015, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001370525686070323, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24595795571804047, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.012373333796858788, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8152620196342468, "metrics/particle/momentum/class_3/accepted/samples": 12404.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22973690927028656, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.590542197227478, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.459624171257019, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3082964420318604, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13601183891296387, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.960066795349121, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007335621397942305, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5043718218803406, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0025958516635000706, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3006458282470703, "metrics/particle/momentum/class_3/assigned/samples": 14859.0, "metrics/particle/momentum/class_3/efficiency": 0.8156769871711731, "metrics/particle/momentum/class_3/event/energy_bias": 3.261662483215332, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.357230186462402, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.5276384353637695, "metrics/particle/momentum/class_3/event/px_bias": 0.14018289744853973, "metrics/particle/momentum/class_3/event/py_bias": 0.1568213254213333, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.000813961029053, "metrics/particle/momentum/class_3/event/pz_bias": -0.6944668292999268, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.14550495147705, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024315394461154938, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14375309646129608, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13774685561656952, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.048479385673999786, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18104059994220734, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010638297535479069, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7022528648376465, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.24453599750995636, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7817058563232422, "metrics/particle/momentum/class_4/accepted/samples": 282.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3144126236438751, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.993277072906494, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.8767940998077393, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7692999839782715, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15833333134651184, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.389404296875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011111111380159855, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9761269688606262, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.044050704687833786, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.5707452297210693, "metrics/particle/momentum/class_4/assigned/samples": 360.0, "metrics/particle/momentum/class_4/efficiency": 0.7747252583503723, "metrics/particle/momentum/class_4/event/energy_bias": -0.2673419713973999, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4816365242004395, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6698954105377197, "metrics/particle/momentum/class_4/event/px_bias": 0.07643359899520874, "metrics/particle/momentum/class_4/event/py_bias": -0.18209968507289886, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.538520336151123, "metrics/particle/momentum/class_4/event/pz_bias": 0.4256565570831299, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.08099889755249, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015536630526185036, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1446504443883896, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14363695681095123, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03689414635300636, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18506957590579987, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026490066200494766, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7948561906814575, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.7559605836868286, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.530564785003662, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.043478116393089294, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16760821640491486, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16342003643512726, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.042747605592012405, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03164556995034218, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22648985683918, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7683731317520142, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7346600890159607, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.464808225631714, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9496855139732361, "metrics/particle/momentum/class_5/event/energy_bias": -0.010084668174386024, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9217339754104614, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.391526222229004, "metrics/particle/momentum/class_5/event/px_bias": -0.15525153279304504, "metrics/particle/momentum/class_5/event/py_bias": 0.2508925795555115, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0745347738265991, "metrics/particle/momentum/class_5/event/pz_bias": -0.3538858890533447, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.905817747116089, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.31813496351242065, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3198474049568176, "metrics/set_slot/angular_recall_all_dr0p10": 0.9602717161178589, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8616991639137268, "metrics/set_slot/soft_count_bias_mean": 0.17161202430725098}, "step_train_time": 5.041766405105591, "step_valid_time": 17930.297902584076, "step_total_time": 17935.3396692276, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002312880160964, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11423573211174509, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2881, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9593657086223984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9590488771466315, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9592072667217176, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04095112285336856, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9517674264948794, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514531043593131, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9516102394715111, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9593657086223984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976926408980437, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11082823731785707, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2263, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2153, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2150, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9518125552608311, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9513919575784358, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516022099447514, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048608042421564294, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504862953138815, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500662836942112, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9502762430939227, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9518125552608311, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040659526583529, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11686367633090955, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9415887850467289, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9401244167962675, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05841121495327103, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.937111801242236, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9400311526479751, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9385692068429238, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386645962732919} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_36000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb37521ea903fd4c1092a9279b4f470c141728e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.7055144968732234, "train": {"Classification_binary": 0.1361393928527832, "Cardinality": 0.096831776201725, "Classification": 0.06718819588422775, "Regression_pt": 0.09619256108999252, "Regression_eta": 0.001102596172131598, "Regression_sin_phi": 0.0007580781821161509, "Regression_cos_phi": 0.0006350893527269363, "Regression_energy": 0.09503204375505447, "HitGrouping": 0.14501428604125977, "Auxiliary": 0.1728929728269577, "Total": 0.8270570635795593, "Total_unweighted": 0.39704795728903264}, "train_group": {}, "valid": {"Classification_binary": 0.13651312390963236, "Cardinality": 0.09718879063924153, "Classification": 0.06744779646396637, "Regression_pt": 0.09596784909566243, "Regression_eta": 0.0012082865772147973, "Regression_sin_phi": 0.0007269755781938633, "Regression_cos_phi": 0.0007337642212708791, "Regression_energy": 0.09542982776959737, "HitGrouping": 0.1443666915098826, "Auxiliary": 0.1745649774869283, "Total": 0.8372282981872559, "Total_unweighted": 0.39802762361553806, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8489631414413452, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6105843782424927, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2827996611595154, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6159995794296265, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6246410608291626, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3333333432674408, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.797088086605072, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5584266781806946, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3076923191547394, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5639181733131409, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5783765316009521, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4171467423439026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.858575165271759, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5941477417945862, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2694453001022339, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5980054140090942, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6038709282875061, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3157731592655182, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8499669432640076, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5962328314781189, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27717325091362, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6007383465766907, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6082678437232971, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3305167257785797, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026480423286557198, "metrics/hit_grouping/calorimeter/background_recall": 0.6476888060569763, "metrics/hit_grouping/calorimeter/completeness": 0.6674132943153381, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6787525415420532, "metrics/hit_grouping/calorimeter/energy_purity": 0.6709880232810974, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3150366246700287, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8676784634590149, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6027499437332153, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22102120518684387, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6101050972938538, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6090950965881348, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2809196710586548, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8533026576042175, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.620397686958313, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.28614750504493713, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6251901984214783, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6335756778717041, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3327268362045288, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8090013861656189, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5572564601898193, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3866767883300781, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5554307103157043, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5801435112953186, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43319427967071533, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8378250002861023, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6373934149742126, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3259420692920685, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6448445320129395, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6663942337036133, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.37209591269493103, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8290450572967529, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7038955092430115, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4097331166267395, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7014549970626831, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7125480771064758, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3678702116012573, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8656975030899048, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5382058024406433, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2077808976173401, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5404332876205444, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5324342250823975, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2810365557670593, "metrics/hit_grouping/calorimeter/purity": 0.6580501794815063, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8094804286956787, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.15280364453792572, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11692307889461517, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15200182795524597, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.11602466553449631, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.029230769723653793, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8188068866729736, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3163640797138214, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11231470853090286, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.315933495759964, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3199780285358429, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24116306006908417, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8526219725608826, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6217811703681946, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2900708317756653, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6266835331916809, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6353408098220825, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3420776426792145, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7538005709648132, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38433951139450073, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.546370804309845, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5977587103843689, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5437836050987244, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8045110702514648, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28266966342926025, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5728534460067749, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6052877902984619, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4223219156265259, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6854372024536133, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4115145206451416, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.46427565813064575, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5233936905860901, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6479483246803284, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7170731425285339, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39207443594932556, "metrics/hit_grouping/tracker/all/object_completeness": 0.49936798214912415, "metrics/hit_grouping/tracker/all/object_purity": 0.5537954568862915, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5960230231285095, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005176037084311247, "metrics/hit_grouping/tracker/background_recall": 0.04756263643503189, "metrics/hit_grouping/tracker/completeness": 0.5147472023963928, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.41685622930526733, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7562493085861206, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37558087706565857, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.576894998550415, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6115241050720215, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.545056939125061, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6951708793640137, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.40719589591026306, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4695473611354828, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.537826418876648, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6320198178291321, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6533784866333008, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4126213490962982, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3589530885219574, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.44261306524276733, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6709142327308655, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7152929306030273, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.39899352192878723, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4993315041065216, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.571897566318512, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5902228355407715, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.691133975982666, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.3921438157558441, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.456827312707901, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5626052618026733, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6351531147956848, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.723953366279602, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38304847478866577, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5068957209587097, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.528664231300354, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5966986417770386, "metrics/hit_grouping/tracker/purity": 0.4907805919647217, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.892058789730072, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05473501235246658, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.15262888371944427, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15276522934436798, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07167680561542511, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6960605978965759, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25920164585113525, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.37444189190864563, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.41045573353767395, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.537065863609314, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6956881284713745, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4530429244041443, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5607088804244995, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6246103048324585, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6739861965179443, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5951938629150391, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7342084646224976, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8352886438369751, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6645814776420593, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10946653038263321, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.13377675414085388, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.55663299560547, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29901543259620667, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16857562959194183, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9247922301292419, "metrics/particle/class_1/pid_accuracy": 0.9780697226524353, "metrics/particle/class_2/efficiency": 0.5619250535964966, "metrics/particle/class_2/pid_accuracy": 0.7797963619232178, "metrics/particle/class_3/efficiency": 0.8122626543045044, "metrics/particle/class_3/pid_accuracy": 0.978131115436554, "metrics/particle/class_4/efficiency": 0.7661212682723999, "metrics/particle/class_4/pid_accuracy": 0.5665829181671143, "metrics/particle/class_5/efficiency": 0.9559999704360962, "metrics/particle/class_5/pid_accuracy": 0.6569037437438965, "metrics/particle/count/bias_mean": -1.0110677480697632, "metrics/particle/count/mae": 3.0006511211395264, "metrics/particle/count/prediction_mean": 69.044921875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05804945528507233, "metrics/particle/event/energy_response_mean": 0.9727680087089539, "metrics/particle/event/met_abs_error": 5.419473171234131, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0446932278573513, "metrics/particle/event/scalar_pt_response_mean": 0.9868538975715637, "metrics/particle/event/vector_pt_closure": 0.04677952826023102, "metrics/particle/matched/delta_eta_abs_mean": 0.01264490932226181, "metrics/particle/matched/delta_phi_abs_mean": 0.013530327938497066, "metrics/particle/matched/delta_r_mean": 0.020815705880522728, "metrics/particle/matched/energy_relative_abs_mean": 0.12325078248977661, "metrics/particle/matched/pid_accuracy": 0.9618744850158691, "metrics/particle/matched/pt_relative_abs_mean": 0.12341025471687317, "metrics/particle/matching/duplicate_fraction": 0.04531696438789368, "metrics/particle/matching/efficiency": 0.844599723815918, "metrics/particle/matching/f1": 0.8507388234138489, "metrics/particle/matching/purity": 0.8569677472114563, "metrics/particle/matching/target_coverage_dr0p05": 0.8023809194564819, "metrics/particle/matching/target_coverage_dr0p10": 0.8809081315994263, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6860305070877075, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8742449283599854, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020677240565419197, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12299951165914536, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12300217896699905, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01595735177397728, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16284792125225067, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006318082567304373, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5974564552307129, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.020727263763546944, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6797417402267456, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9180.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2001364380121231, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0160108804702759, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1436772346496582, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0100970268249512, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09881796687841415, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 21.727109909057617, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012482269667088985, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8667129278182983, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.052401103079319, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6561105251312256, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10575.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8394293785095215, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.519242286682129, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.109736442565918, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.121239423751831, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.05066147446632385, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07870876789093018, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.112544536590576, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.08815394341945648, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.857261657714844, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.020860711112618446, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13261102139949799, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13255855441093445, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01989058591425419, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17322424054145813, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026070764288306236, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3268020153045654, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04118175059556961, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5690340995788574, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2148.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.38296955823898315, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1840332746505737, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.07585072517395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9250319004058838, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.15000000596046448, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.2414493560791, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.056343283504247665, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2469725608825684, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.14372587203979492, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.658506393432617, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2680.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7110228538513184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.451138496398926, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.490691184997559, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2728700637817383, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06703117489814758, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07299523800611496, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.33066177368164, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6994462013244629, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.556720733642578, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02056698314845562, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12205850332975388, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12192527204751968, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014959578402340412, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16234491765499115, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005405697738751769, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20047150552272797, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005225213710218668, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6468073725700378, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18499.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15634360909461975, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7989442348480225, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6349671483039856, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4975854456424713, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09590732306241989, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.0397210121154785, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027062958106398582, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33150696754455566, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0024972916580736637, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3120269775390625, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21062.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8629472255706787, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.454192876815796, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.436433792114258, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.274103164672852, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.19172605872154236, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.260727882385254, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.3147244453430176, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7606459856033325, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.928837776184082, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020622074604034424, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12310806661844254, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12302248179912567, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015621776692569256, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1633068472146988, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004157307092100382, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4037665128707886, "metrics/particle/momentum/all/accepted/pz_bias": 0.006654344033449888, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.430104374885559, "metrics/particle/momentum/all/accepted/samples": 29827.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1875370889902115, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8959083557128906, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9042556881904602, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7669960856437683, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10102864354848862, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.137041091918945, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009907625615596771, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6460230946540833, "metrics/particle/momentum/all/assigned/pz_bias": 0.006456107832491398, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.587073802947998, "metrics/particle/momentum/all/assigned/samples": 34317.0, "metrics/particle/momentum/all/efficiency": 0.842713475227356, "metrics/particle/momentum/all/event/energy_bias": -8.42457389831543, "metrics/particle/momentum/all/event/p3_error_mean": 16.41657829284668, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.743753910064697, "metrics/particle/momentum/all/event/px_bias": -0.2080957144498825, "metrics/particle/momentum/all/event/py_bias": -1.2550143003463745, "metrics/particle/momentum/all/event/pz_abs_mean": 12.315269470214844, "metrics/particle/momentum/all/event/pz_bias": -0.026954352855682373, "metrics/particle/momentum/all/event/pz_residual_rms": 21.8024845123291, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02186545915901661, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09672573953866959, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09667744487524033, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008910867385566235, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1312107890844345, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0038855071179568768, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4039112627506256, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0018591274274513125, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.370469331741333, "metrics/particle/momentum/class_1/accepted/samples": 15442.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09509684145450592, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21414496004581451, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23340362310409546, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12455805391073227, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.041684363037347794, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.208928108215332, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00667435210198164, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5106666684150696, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.004572256002575159, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.8263823986053467, "metrics/particle/momentum/class_1/assigned/samples": 16481.0, "metrics/particle/momentum/class_1/efficiency": 0.9253910183906555, "metrics/particle/momentum/class_1/event/energy_bias": 1.4171476364135742, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.870388984680176, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.140706062316895, "metrics/particle/momentum/class_1/event/px_bias": -0.4528205692768097, "metrics/particle/momentum/class_1/event/py_bias": -0.5726912021636963, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.618138313293457, "metrics/particle/momentum/class_1/event/pz_bias": -0.6342217326164246, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.97522735595703, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027466077357530594, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17263685166835785, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1736936718225479, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.014730177819728851, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21382130682468414, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.023679416626691818, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4005604982376099, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.07156401872634888, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5479795932769775, "metrics/particle/momentum/class_2/accepted/samples": 1647.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.567986249923706, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6507578492164612, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8395813703536987, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5097501277923584, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2501989006996155, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.039504051208496, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04653938114643097, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2967185974121094, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0029577407985925674, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.780049800872803, "metrics/particle/momentum/class_2/assigned/samples": 2514.0, "metrics/particle/momentum/class_2/efficiency": 0.553241491317749, "metrics/particle/momentum/class_2/event/energy_bias": -11.055503845214844, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.21667766571045, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.467766761779785, "metrics/particle/momentum/class_2/event/px_bias": 0.22923538088798523, "metrics/particle/momentum/class_2/event/py_bias": -0.5934280157089233, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.953229904174805, "metrics/particle/momentum/class_2/event/pz_bias": -1.0395917892456055, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.882917404174805, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01815607026219368, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1489885300397873, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1487429291009903, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027576850727200508, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18915247917175293, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013814399717375636, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2460610717535019, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.019458623602986336, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8284909129142761, "metrics/particle/momentum/class_3/accepted/samples": 12306.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22508545219898224, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6898179054260254, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6577600240707397, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.52153742313385, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14113038778305054, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.26875114440918, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006887703202664852, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49687379598617554, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0171594750136137, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.39784836769104, "metrics/particle/momentum/class_3/assigned/samples": 14809.0, "metrics/particle/momentum/class_3/efficiency": 0.8092325925827026, "metrics/particle/momentum/class_3/event/energy_bias": 1.8115017414093018, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.689727783203125, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.385007381439209, "metrics/particle/momentum/class_3/event/px_bias": -0.08412755280733109, "metrics/particle/momentum/class_3/event/py_bias": -0.36804524064064026, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.472641468048096, "metrics/particle/momentum/class_3/event/pz_bias": 0.9468600749969482, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.534611701965332, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02296561561524868, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1435861885547638, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14320451021194458, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04652774706482887, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18638703227043152, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0071428571827709675, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7416876554489136, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.28881245851516724, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8490779399871826, "metrics/particle/momentum/class_4/accepted/samples": 280.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26440972089767456, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.4955555200576782, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4095104932785034, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2898176908493042, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1525423675775528, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.550227642059326, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9521856904029846, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.22942645847797394, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1218464374542236, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7692307829856873, "metrics/particle/momentum/class_4/event/energy_bias": -0.2554508447647095, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.724591016769409, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7567963600158691, "metrics/particle/momentum/class_4/event/px_bias": 0.2935822010040283, "metrics/particle/momentum/class_4/event/py_bias": -0.016263462603092194, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7783254384994507, "metrics/particle/momentum/class_4/event/pz_bias": 0.7285057902336121, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.384897708892822, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01547668781131506, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13365231454372406, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13090240955352783, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0014556142268702388, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1679375171661377, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03947368264198303, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7337015867233276, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.21505840122699738, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5003209114074707, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08560065925121307, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16112852096557617, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1583411544561386, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.015040508471429348, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.044025156646966934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22963783144950867, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.044025156646966934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7864766120910645, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2884136438369751, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6511216163635254, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": -0.3422694504261017, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7394319772720337, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3778645992279053, "metrics/particle/momentum/class_5/event/px_bias": -0.19396521151065826, "metrics/particle/momentum/class_5/event/py_bias": 0.295413613319397, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8394728899002075, "metrics/particle/momentum/class_5/event/pz_bias": -0.0285068079829216, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7148327827453613, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3088362514972687, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3102988302707672, "metrics/set_slot/angular_recall_all_dr0p10": 0.9619259238243103, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8656394481658936, "metrics/set_slot/soft_count_bias_mean": 0.18298034369945526}, "step_train_time": 4.947448968887329, "step_valid_time": 21043.361397743225, "step_total_time": 21048.30884718895, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933037560777541, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11112275128740123, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3018, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2899, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9577139081598943, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9605699138502319, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591397849462365, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039430086149768055, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520977865873802, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9549370444002651, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9535153019023986, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9577139081598943, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897152584879589, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10867304071805572, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2255, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2152, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2150, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9513704686118479, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543237250554324, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528448085012177, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04567627494456763, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9504862953138815, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534368070953437, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.951959264998893, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9513704686118479, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9887820912562497, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10345920401102637, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362363919129082, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9355089355089355, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06376360808709176, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9324534161490683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9339035769828927, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9331779331779332, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347826086956522} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_37000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..7d2837a39dbc96fcac6a629c4a6e9e2f582e2d0e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7528898995641462, "train": {"Classification_binary": 0.1465064138174057, "Cardinality": 0.1181902289390564, "Classification": 0.0635603815317154, "Regression_pt": 0.0958704799413681, "Regression_eta": 0.00120511744171381, "Regression_sin_phi": 0.0007009110413491726, "Regression_cos_phi": 0.0006542280316352844, "Regression_energy": 0.09543941915035248, "HitGrouping": 0.14862599968910217, "Auxiliary": 0.18031351268291473, "Total": 0.8678699731826782, "Total_unweighted": 0.40393695095553994}, "train_group": {}, "valid": {"Classification_binary": 0.1363143026828766, "Cardinality": 0.0968572199344635, "Classification": 0.0675840179125468, "Regression_pt": 0.09389022986094157, "Regression_eta": 0.001224707734460632, "Regression_sin_phi": 0.0007171389491607746, "Regression_cos_phi": 0.0007355785928666592, "Regression_energy": 0.09342758854230244, "HitGrouping": 0.14550462365150452, "Auxiliary": 0.17409916718800864, "Total": 0.8351655006408691, "Total_unweighted": 0.3938935642751555, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8472243547439575, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6101080179214478, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2890337109565735, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6157437562942505, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6208161115646362, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3407008647918701, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7995412945747375, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5583315491676331, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30870890617370605, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.565991997718811, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.572852373123169, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4046086072921753, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8571723103523254, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5882113575935364, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27316391468048096, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5917801856994629, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5979946255683899, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32036980986595154, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8488019108772278, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5925773978233337, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2814589738845825, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5971783995628357, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6030832529067993, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3344680964946747, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026548517867922783, "metrics/hit_grouping/calorimeter/background_recall": 0.6478615403175354, "metrics/hit_grouping/calorimeter/completeness": 0.6648718118667603, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6754201650619507, "metrics/hit_grouping/calorimeter/energy_purity": 0.6677209734916687, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3164422810077667, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8663988709449768, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5965231657028198, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22430576384067535, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6042052507400513, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.599571704864502, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28378620743751526, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8523117899894714, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6183829307556152, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2894226014614105, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6220102310180664, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6318804025650024, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.338064044713974, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8078972101211548, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5573339462280273, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3941347599029541, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5564278960227966, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5805068612098694, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4379977285861969, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8372765779495239, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6358726620674133, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32793521881103516, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6426002383232117, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6613876819610596, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.37689194083213806, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8246028423309326, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6990101933479309, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.42804813385009766, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6959869861602783, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7118245959281921, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37571951746940613, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8642888069152832, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5324128866195679, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21273604035377502, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.535703182220459, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.526487410068512, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28358107805252075, "metrics/hit_grouping/calorimeter/purity": 0.6555902361869812, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8044096231460571, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.13738662004470825, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10615384578704834, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1348058581352234, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.09915892034769058, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03384615480899811, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8182680010795593, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29785606265068054, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10547320544719696, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.29769524931907654, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2939132750034332, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23945267498493195, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8515042066574097, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6192306280136108, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2953174114227295, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6242585182189941, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6316062211990356, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34634050726890564, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7518957257270813, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4036162495613098, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5397568941116333, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5795231461524963, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5515540838241577, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8041214346885681, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30061694979667664, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5745407342910767, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5948573350906372, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40998318791389465, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6819525361061096, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42065054178237915, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4585281014442444, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5117745995521545, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6555091738700867, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7143484354019165, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40515774488449097, "metrics/hit_grouping/tracker/all/object_completeness": 0.49397268891334534, "metrics/hit_grouping/tracker/all/object_purity": 0.5401852130889893, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6019743084907532, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005795209668576717, "metrics/hit_grouping/tracker/background_recall": 0.04471953958272934, "metrics/hit_grouping/tracker/completeness": 0.5106536149978638, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.43890580534935, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7509045004844666, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38314351439476013, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5655035376548767, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5944775342941284, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5534396171569824, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6922159194946289, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42052161693573, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.46890226006507874, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5302531719207764, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6381115317344666, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6567147970199585, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4377022683620453, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.36182162165641785, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4302164316177368, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.671318769454956, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7132143974304199, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41651690006256104, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4946240484714508, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5560845732688904, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.598490297794342, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6906957030296326, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41544607281684875, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4679812788963318, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5582944750785828, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6331557631492615, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7199848294258118, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38855069875717163, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4976945221424103, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5162882804870605, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6010302305221558, "metrics/hit_grouping/tracker/purity": 0.48718082904815674, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8925033807754517, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04430929571390152, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13749192655086517, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.14146173000335693, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07949609309434891, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7075425386428833, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28564023971557617, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.38266751170158386, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.39886003732681274, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5209953188896179, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6909151077270508, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4678000211715698, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5550863742828369, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.610457181930542, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6821911334991455, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5849410891532898, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7078300714492798, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8033239841461182, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6664626598358154, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09921381622552872, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1356579214334488, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.81374740600586, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29649993777275085, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16606009006500244, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9200435280799866, "metrics/particle/class_1/pid_accuracy": 0.9749457240104675, "metrics/particle/class_2/efficiency": 0.5621494054794312, "metrics/particle/class_2/pid_accuracy": 0.7970464825630188, "metrics/particle/class_3/efficiency": 0.8250488042831421, "metrics/particle/class_3/pid_accuracy": 0.9783920049667358, "metrics/particle/class_4/efficiency": 0.7718960642814636, "metrics/particle/class_4/pid_accuracy": 0.6084787845611572, "metrics/particle/class_5/efficiency": 0.9480000138282776, "metrics/particle/class_5/pid_accuracy": 0.6835442781448364, "metrics/particle/count/bias_mean": -0.6100260615348816, "metrics/particle/count/mae": 2.974609375, "metrics/particle/count/prediction_mean": 69.44596099853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05789860710501671, "metrics/particle/event/energy_response_mean": 0.9758316874504089, "metrics/particle/event/met_abs_error": 5.4971089363098145, "metrics/particle/event/scalar_pt_relative_abs_error": 0.044445812702178955, "metrics/particle/event/scalar_pt_response_mean": 0.9929444193840027, "metrics/particle/event/vector_pt_closure": 0.04471905156970024, "metrics/particle/matched/delta_eta_abs_mean": 0.012702519074082375, "metrics/particle/matched/delta_phi_abs_mean": 0.013707845471799374, "metrics/particle/matched/delta_r_mean": 0.020963378250598907, "metrics/particle/matched/energy_relative_abs_mean": 0.1248542070388794, "metrics/particle/matched/pid_accuracy": 0.9618933796882629, "metrics/particle/matched/pt_relative_abs_mean": 0.12532417476177216, "metrics/particle/matching/duplicate_fraction": 0.04324593022465706, "metrics/particle/matching/efficiency": 0.8479452729225159, "metrics/particle/matching/f1": 0.8516532778739929, "metrics/particle/matching/purity": 0.8553937673568726, "metrics/particle/matching/target_coverage_dr0p05": 0.8010055422782898, "metrics/particle/matching/target_coverage_dr0p10": 0.8824508190155029, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.686755359172821, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.878668487071991, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02100113406777382, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12451902776956558, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12449215352535248, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019700627774000168, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16251619160175323, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005204944871366024, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5944426655769348, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.012468164786696434, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7103924751281738, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9222.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19519563019275665, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8404936194419861, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.840551495552063, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7035303115844727, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09762533009052277, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.66645622253418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011402186006307602, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8444788455963135, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0056789834052324295, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.510385513305664, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10612.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.843269944190979, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6236205697059631, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.374689102172852, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1611342430114746, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.19360464811325073, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.19134521484375, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.383496284484863, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0826663970947266, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.379457473754883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02177073247730732, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13475994765758514, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13817982375621796, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02763422392308712, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17769508063793182, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02867889031767845, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3524956703186035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1503429114818573, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.705876588821411, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2127.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.41202303767204285, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0373010635375977, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1931605339050293, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.026707172393799, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1606217622756958, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 22.210033416748047, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06328645348548889, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3861796855926514, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.16270995140075684, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.304495811462402, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2702.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7040715217590332, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.8926875591278076, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.4141130447387695, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.148972988128662, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12725293636322021, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.049711648374795914, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.254147529602051, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.8979887962341309, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.11957359313965, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020546190440654755, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12307405471801758, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12345021218061447, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017073217779397964, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1624191850423813, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003759600513149053, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19590522348880768, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005605415441095829, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6424200534820557, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18619.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15465839207172394, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7664902806282043, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5968521237373352, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.456417977809906, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09158392250537872, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.201413154602051, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028368793427944183, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3290146291255951, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0032758978195488453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4682815074920654, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21150.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8685450553894043, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9031096696853638, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.598921775817871, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.508399486541748, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.370166540145874, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.21424554288387299, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2980270385742188, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.17750553786754608, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.553470611572266, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02077310159802437, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12434813380241394, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12481629103422165, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01863132417201996, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16358010470867157, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0038707954809069633, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4006362557411194, "metrics/particle/momentum/all/accepted/pz_bias": -0.00335126556456089, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4599310159683228, "metrics/particle/momentum/all/accepted/samples": 29968.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18731799721717834, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8105087876319885, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7970424890518188, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6556194424629211, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09885677695274353, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.864442825317383, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01021355576813221, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6490171551704407, "metrics/particle/momentum/all/assigned/pz_bias": -0.008997553028166294, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5258007049560547, "metrics/particle/momentum/all/assigned/samples": 34464.0, "metrics/particle/momentum/all/efficiency": 0.8466971516609192, "metrics/particle/momentum/all/event/energy_bias": -5.419417381286621, "metrics/particle/momentum/all/event/p3_error_mean": 16.350902557373047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.05636215209961, "metrics/particle/momentum/all/event/px_bias": 0.43651828169822693, "metrics/particle/momentum/all/event/py_bias": 0.35587912797927856, "metrics/particle/momentum/all/event/pz_abs_mean": 12.134114265441895, "metrics/particle/momentum/all/event/pz_bias": -2.15816068649292, "metrics/particle/momentum/all/event/pz_residual_rms": 19.11981773376465, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02200545370578766, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10036910325288773, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1019192487001419, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02654317580163479, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1363346427679062, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004038561601191759, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.41376104950904846, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007543627172708511, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4414479732513428, "metrics/particle/momentum/class_1/accepted/samples": 15352.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10032016038894653, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2301478534936905, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2470654547214508, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.14257408678531647, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04526449739933014, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4487464427948, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006726049818098545, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5292962789535522, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0016946196556091309, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9494483470916748, "metrics/particle/momentum/class_1/assigned/samples": 16503.0, "metrics/particle/momentum/class_1/efficiency": 0.9199976325035095, "metrics/particle/momentum/class_1/event/energy_bias": 1.7113761901855469, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.572831153869629, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.159416198730469, "metrics/particle/momentum/class_1/event/px_bias": 0.29349130392074585, "metrics/particle/momentum/class_1/event/py_bias": 0.05606117844581604, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.246689796447754, "metrics/particle/momentum/class_1/event/pz_bias": -0.24863654375076294, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.46108627319336, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02736632525920868, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17250481247901917, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17280182242393494, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.00823341403156519, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21426348388195038, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021276595070958138, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4075196981430054, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08708732575178146, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6661295890808105, "metrics/particle/momentum/class_2/accepted/samples": 1645.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5705628395080566, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6147608757019043, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8683854341506958, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5306784510612488, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26361143589019775, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.596073627471924, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04778691753745079, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.248141050338745, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.007414671592414379, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.676074028015137, "metrics/particle/momentum/class_2/assigned/samples": 2553.0, "metrics/particle/momentum/class_2/efficiency": 0.5525696873664856, "metrics/particle/momentum/class_2/event/energy_bias": -8.672480583190918, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.306991577148438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.005221366882324, "metrics/particle/momentum/class_2/event/px_bias": 0.5243191719055176, "metrics/particle/momentum/class_2/event/py_bias": 0.001625198870897293, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.63797664642334, "metrics/particle/momentum/class_2/event/pz_bias": -1.3057729005813599, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.78303337097168, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018435025587677956, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14738035202026367, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1465567797422409, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.009649540297687054, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18508081138134003, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011962676653638482, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.234893336892128, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.010012451559305191, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7926665544509888, "metrics/particle/momentum/class_3/accepted/samples": 12539.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21883845329284668, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4707881212234497, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3949543237686157, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2446434497833252, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12965822219848633, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.565999984741211, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007453165948390961, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4902830719947815, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.017243176698684692, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1467316150665283, "metrics/particle/momentum/class_3/assigned/samples": 14893.0, "metrics/particle/momentum/class_3/efficiency": 0.8245545029640198, "metrics/particle/momentum/class_3/event/energy_bias": 1.4280375242233276, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.985710144042969, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.051468849182129, "metrics/particle/momentum/class_3/event/px_bias": -0.09092455357313156, "metrics/particle/momentum/class_3/event/py_bias": 0.1777215301990509, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.961222171783447, "metrics/particle/momentum/class_3/event/pz_bias": -0.3769875764846802, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.81624984741211, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022634675726294518, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1306895613670349, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13104771077632904, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03688503056764603, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17166858911514282, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0071428571827709675, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7115108370780945, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.28982940316200256, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9203904867172241, "metrics/particle/momentum/class_4/accepted/samples": 280.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.21150153875350952, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7390832901000977, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.01060152053833, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9085878729820251, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14565826952457428, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.370280742645264, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016806723549962044, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0840567350387573, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04403966665267944, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.84529185295105, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.7692307829856873, "metrics/particle/momentum/class_4/event/energy_bias": -0.2546236515045166, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.7266719341278076, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7395774126052856, "metrics/particle/momentum/class_4/event/px_bias": -0.009963667020201683, "metrics/particle/momentum/class_4/event/py_bias": -0.06439090520143509, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7757549285888672, "metrics/particle/momentum/class_4/event/pz_bias": 0.08356207609176636, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.729637622833252, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014398052357137203, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11337275058031082, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.11317425966262817, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.039376504719257355, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.14630581438541412, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.01315789483487606, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.278211236000061, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.31618860363960266, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.506621837615967, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05588456615805626, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2560878098011017, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24762679636478424, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.16901233792304993, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.547370195388794, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.012658228166401386, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.2939954996109009, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29495877027511597, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.4996848106384277, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": 0.3682730197906494, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6150743961334229, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.178839087486267, "metrics/particle/momentum/class_5/event/px_bias": -0.2804040014743805, "metrics/particle/momentum/class_5/event/py_bias": 0.18486209213733673, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8713248372077942, "metrics/particle/momentum/class_5/event/pz_bias": -0.3103257715702057, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3871090412139893, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3053194582462311, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3063647150993347, "metrics/set_slot/angular_recall_all_dr0p10": 0.9617307782173157, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8639388084411621, "metrics/set_slot/soft_count_bias_mean": 0.25137564539909363}, "step_train_time": 6.653772830963135, "step_valid_time": 24041.349868774414, "step_total_time": 24048.003641843796, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989037178451929, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10845741578087209, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2873, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9570531879748926, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9570531879748926, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9570531879748926, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04294681202510737, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9491245457548728, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9491245457548728, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9491245457548728, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9570531879748926, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9923904395875442, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11351825266462945, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2149, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2145, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9500442086648984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542628774422736, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521488701816571, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04573712255772647, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9482758620689655, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9524866785079928, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503766061143111, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9500442086648984, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9915244828995188, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10267819943941436, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1207, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.937111801242236, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9407638347622759, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9389342668222481, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059236165237724084, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355590062111802, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9392049883086516, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9373784519642162, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.937111801242236} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_38000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..6da754cc7df57e5211a6bf8f0b7fd01e29126b32 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.8002653022550692, "train": {"Classification_binary": 0.1386529952287674, "Cardinality": 0.0925130546092987, "Classification": 0.06349743157625198, "Regression_pt": 0.09130232781171799, "Regression_eta": 0.0016418196028098464, "Regression_sin_phi": 0.0007327315979637206, "Regression_cos_phi": 0.0007205680594779551, "Regression_energy": 0.09148827940225601, "HitGrouping": 0.1423323154449463, "Auxiliary": 0.1686301976442337, "Total": 0.8315571546554565, "Total_unweighted": 0.3880361532792449}, "train_group": {}, "valid": {"Classification_binary": 0.13557463884353638, "Cardinality": 0.09668022394180298, "Classification": 0.06577294071515401, "Regression_pt": 0.09396785497665405, "Regression_eta": 0.0012032546413441498, "Regression_sin_phi": 0.0006792502632985512, "Regression_cos_phi": 0.0007006043257812659, "Regression_energy": 0.09339184562365214, "HitGrouping": 0.14481258392333984, "Auxiliary": 0.17316321531931558, "Total": 0.8255360921223959, "Total_unweighted": 0.39129038938942057, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8493886590003967, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6105800271034241, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28733351826667786, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6159548163414001, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.621496856212616, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3331444263458252, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8000394105911255, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.558955729007721, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3043036162853241, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5651868581771851, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5753637552261353, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.40664181113243103, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8610022068023682, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5931259989738464, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2708914279937744, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5967655181884766, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5997841358184814, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3114347755908966, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8517968654632568, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5956776738166809, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27917933464050293, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6001080274581909, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6045807600021362, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3269604742527008, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026501471176743507, "metrics/hit_grouping/calorimeter/background_recall": 0.6481144428253174, "metrics/hit_grouping/calorimeter/completeness": 0.6690676212310791, "metrics/hit_grouping/calorimeter/energy_completeness": 0.680182695388794, "metrics/hit_grouping/calorimeter/energy_purity": 0.6724469065666199, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.31805938482284546, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8693087100982666, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6010362505912781, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22466407716274261, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6088821887969971, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6037381291389465, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2780531644821167, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8542793393135071, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6197318434715271, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2832362949848175, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6237361431121826, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6350475549697876, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3327268362045288, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8121432662010193, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.559268593788147, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3903425633907318, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5569133162498474, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5746150612831116, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.42447224259376526, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8410476446151733, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6402636170387268, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32681408524513245, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6469205617904663, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6643834710121155, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36181876063346863, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8302751779556274, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7048040628433228, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4207221269607544, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7003244757652283, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7054212689399719, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.36368393898010254, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8661071062088013, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5337807536125183, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20985670387744904, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5369576215744019, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5273851752281189, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2847864031791687, "metrics/hit_grouping/calorimeter/purity": 0.659695565700531, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8217881321907043, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.16822215914726257, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10769230872392654, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.16535165905952454, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12123018503189087, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.032307691872119904, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8245692849159241, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3242250084877014, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11801595985889435, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3227519094944, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.31255045533180237, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2491448074579239, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8540025353431702, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6204012632369995, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2921038866043091, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6253268718719482, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6316792964935303, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3377164304256439, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7482793927192688, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40107589960098267, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5440464615821838, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5874112844467163, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5531978607177734, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8069587349891663, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.27369600534439087, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5877445340156555, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6216855049133301, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41895681619644165, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6832680106163025, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4209655821323395, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.46728983521461487, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5164672136306763, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6488146781921387, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7142332196235657, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4022766053676605, "metrics/hit_grouping/tracker/all/object_completeness": 0.5016949772834778, "metrics/hit_grouping/tracker/all/object_purity": 0.5477520823478699, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5992348194122314, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005654904525727034, "metrics/hit_grouping/tracker/background_recall": 0.05182728171348572, "metrics/hit_grouping/tracker/completeness": 0.5151652693748474, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4276411831378937, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7510427832603455, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37940773367881775, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5749639868736267, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6064821481704712, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5536218881607056, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6902762055397034, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4249000549316406, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4719189405441284, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5383144617080688, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6381115317344666, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6579756736755371, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42900484800338745, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37068480253219604, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4274066686630249, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6591828465461731, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.713426411151886, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40932783484458923, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5017879009246826, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5617817640304565, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5904026031494141, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6943796873092651, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.43608522415161133, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4778352975845337, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5424158573150635, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6225033402442932, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7187754511833191, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38714587688446045, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5057693719863892, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5304132103919983, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6066495180130005, "metrics/hit_grouping/tracker/purity": 0.49141570925712585, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8960400819778442, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05256298929452896, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.15271200239658356, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15078136324882507, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0725456103682518, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.707545280456543, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27630895376205444, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.39309465885162354, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4149197041988373, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5241057276725769, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6902902722358704, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46414026618003845, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.561481773853302, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6168187260627747, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6793577671051025, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.584621012210846, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7284348607063293, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8291828036308289, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6579522490501404, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09889361262321472, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12714751064777374, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.57331085205078, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30010995268821716, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16967011988162994, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9207558631896973, "metrics/particle/class_1/pid_accuracy": 0.976426362991333, "metrics/particle/class_2/efficiency": 0.5493605732917786, "metrics/particle/class_2/pid_accuracy": 0.7925260066986084, "metrics/particle/class_3/efficiency": 0.8189131617546082, "metrics/particle/class_3/pid_accuracy": 0.9815571904182434, "metrics/particle/class_4/efficiency": 0.7709335684776306, "metrics/particle/class_4/pid_accuracy": 0.5942571759223938, "metrics/particle/class_5/efficiency": 0.9580000042915344, "metrics/particle/class_5/pid_accuracy": 0.6701461672782898, "metrics/particle/count/bias_mean": -1.4759114980697632, "metrics/particle/count/mae": 3.1647136211395264, "metrics/particle/count/prediction_mean": 68.580078125, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.056373756378889084, "metrics/particle/event/energy_response_mean": 1.0004252195358276, "metrics/particle/event/met_abs_error": 5.438278675079346, "metrics/particle/event/scalar_pt_relative_abs_error": 0.045934513211250305, "metrics/particle/event/scalar_pt_response_mean": 1.004607081413269, "metrics/particle/event/vector_pt_closure": 0.04615306854248047, "metrics/particle/matched/delta_eta_abs_mean": 0.01257925946265459, "metrics/particle/matched/delta_phi_abs_mean": 0.013072130270302296, "metrics/particle/matched/delta_r_mean": 0.020405352115631104, "metrics/particle/matched/energy_relative_abs_mean": 0.12475841492414474, "metrics/particle/matched/pid_accuracy": 0.9636899828910828, "metrics/particle/matched/pt_relative_abs_mean": 0.1246563047170639, "metrics/particle/matching/duplicate_fraction": 0.04303249344229698, "metrics/particle/matching/efficiency": 0.844599723815918, "metrics/particle/matching/f1": 0.8535913228988647, "metrics/particle/matching/purity": 0.8627763986587524, "metrics/particle/matching/target_coverage_dr0p05": 0.804667055606842, "metrics/particle/matching/target_coverage_dr0p10": 0.879625678062439, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6858260631561279, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8737245202064514, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019968299195170403, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1260698288679123, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12520751357078552, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019801180809736252, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1643487960100174, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005801226012408733, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6021394729614258, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.058271437883377075, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6358847618103027, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9136.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19218051433563232, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8082512021064758, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9060974717140198, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7737839221954346, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09475594758987427, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.411118507385254, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012799694202840328, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8675532341003418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.036260176450014114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6547067165374756, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10469.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8354060053825378, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.15275582671165466, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.512071132659912, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.26956844329834, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2313576638698578, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.44308677315711975, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.521993160247803, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6017358303070068, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.690468788146973, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021030819043517113, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13450923562049866, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13285820186138153, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017110323533415794, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17070500552654266, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023966165259480476, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3166815042495728, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19552862644195557, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6110222339630127, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2128.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3315557837486267, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0081735849380493, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9005277156829834, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.760097861289978, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1380753070116043, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.672693252563477, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.049068085849285126, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.089214563369751, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1889171600341797, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.652162075042725, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2629.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.704402506351471, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.7083721160888672, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.554990291595459, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1633402109146118, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.028933361172676086, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07489273697137833, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.421207904815674, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.2733854055404663, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.274038314819336, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02022005245089531, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12167026102542877, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12202451378107071, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014430706389248371, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16105584800243378, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00043112740968354046, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19712181389331818, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.014962499029934406, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6034340858459473, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18556.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1511574685573578, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7854518294334412, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6054593920707703, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46680235862731934, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09170036017894745, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.018034934997559, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003329369705170393, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3393116593360901, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.004163892008364201, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.659530758857727, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21025.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8656061887741089, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7471320629119873, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.414242744445801, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.315499305725098, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.23007425665855408, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.5095585584640503, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2576043605804443, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.14445775747299194, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.597163200378418, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020200779661536217, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12393437325954437, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12377279996871948, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016267290338873863, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16277505457401276, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0037558686453849077, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.40110114216804504, "metrics/particle/momentum/all/accepted/pz_bias": -0.04111659526824951, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4060553312301636, "metrics/particle/momentum/all/accepted/samples": 29820.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17764216661453247, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8096062541007996, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7974739670753479, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6606267094612122, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.0962107703089714, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.579046249389648, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009758813306689262, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6361980438232422, "metrics/particle/momentum/all/assigned/pz_bias": -0.02311418019235134, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.514047145843506, "metrics/particle/momentum/all/assigned/samples": 34123.0, "metrics/particle/momentum/all/efficiency": 0.8425157070159912, "metrics/particle/momentum/all/event/energy_bias": 0.19151580333709717, "metrics/particle/momentum/all/event/p3_error_mean": 15.870964050292969, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.751479625701904, "metrics/particle/momentum/all/event/px_bias": -0.027649953961372375, "metrics/particle/momentum/all/event/py_bias": 1.8777527809143066, "metrics/particle/momentum/all/event/pz_abs_mean": 11.853104591369629, "metrics/particle/momentum/all/event/pz_bias": -2.0195789337158203, "metrics/particle/momentum/all/event/pz_residual_rms": 19.692270278930664, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021780870854854584, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09887586534023285, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0988689586520195, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009065288119018078, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.13238558173179626, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033856371883302927, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39988380670547485, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07133328914642334, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3091415166854858, "metrics/particle/momentum/class_1/accepted/samples": 15359.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08975304663181305, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23882819712162018, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23690864443778992, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1255674511194229, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.041420478373765945, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.5927822589874268, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005969421938061714, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4992358684539795, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.05623668432235718, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.801945447921753, "metrics/particle/momentum/class_1/assigned/samples": 16417.0, "metrics/particle/momentum/class_1/efficiency": 0.920417070388794, "metrics/particle/momentum/class_1/event/energy_bias": 4.508927822113037, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.251141548156738, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.687923431396484, "metrics/particle/momentum/class_1/event/px_bias": -0.06928213685750961, "metrics/particle/momentum/class_1/event/py_bias": 0.9721099138259888, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.314053535461426, "metrics/particle/momentum/class_1/event/pz_bias": -1.5225622653961182, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.330284118652344, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02703257091343403, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17448578774929047, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17463664710521698, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.007921245880424976, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2145007848739624, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024088943377137184, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4468446969985962, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0007760871085338295, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5005645751953125, "metrics/particle/momentum/class_2/accepted/samples": 1619.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5337251424789429, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7128487825393677, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7942275404930115, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.48061802983283997, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.24918167293071747, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.085995674133301, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05032733082771301, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.353543281555176, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13986320793628693, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.08012056350708, "metrics/particle/momentum/class_2/assigned/samples": 2444.0, "metrics/particle/momentum/class_2/efficiency": 0.5438360571861267, "metrics/particle/momentum/class_2/event/energy_bias": -9.099658966064453, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.570403099060059, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.764030456542969, "metrics/particle/momentum/class_2/event/px_bias": 0.6636286973953247, "metrics/particle/momentum/class_2/event/py_bias": 0.005540016107261181, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.166549682617188, "metrics/particle/momentum/class_2/event/pz_bias": -0.30020761489868164, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.065494537353516, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017313972115516663, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14786534011363983, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14771772921085358, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02571210451424122, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18698319792747498, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001450910815037787, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24780263006687164, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00757169583812356, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9373311400413513, "metrics/particle/momentum/class_3/accepted/samples": 12406.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21690154075622559, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4347299337387085, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4005980491638184, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2638881206512451, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13099193572998047, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.010258674621582, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007119126617908478, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.48696112632751465, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.009604030288755894, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1326494216918945, "metrics/particle/momentum/class_3/assigned/samples": 14749.0, "metrics/particle/momentum/class_3/efficiency": 0.8158085346221924, "metrics/particle/momentum/class_3/event/energy_bias": 4.8589863777160645, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.805350303649902, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.629377365112305, "metrics/particle/momentum/class_3/event/px_bias": -0.3577762842178345, "metrics/particle/momentum/class_3/event/py_bias": 0.6888070106506348, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.48362398147583, "metrics/particle/momentum/class_3/event/pz_bias": -0.33865877985954285, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.642807960510254, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025130704045295715, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1490662395954132, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13708943128585815, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03853030130267143, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17958851158618927, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0035211266949772835, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6586976647377014, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17582294344902039, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.566900372505188, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.21520043909549713, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1656904220581055, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9472850561141968, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8279473781585693, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15774647891521454, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.246109008789062, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008450704626739025, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9363771080970764, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.05109878256917, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.121572971343994, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.7802197933197021, "metrics/particle/momentum/class_4/event/energy_bias": -0.14485475420951843, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.502577781677246, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6841517686843872, "metrics/particle/momentum/class_4/event/px_bias": -0.03438311815261841, "metrics/particle/momentum/class_4/event/py_bias": -0.0317324697971344, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5701661109924316, "metrics/particle/momentum/class_4/event/pz_bias": 0.294388085603714, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.882304668426514, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014176754280924797, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11739335209131241, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.11921297758817673, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02042980305850506, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.15224923193454742, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.01315789483487606, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4162508249282837, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.5772680044174194, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.8034679889678955, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.052571628242731094, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.21202319860458374, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20936918258666992, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10433847457170486, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.037974681705236435, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.972287654876709, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5593074560165405, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5304012298583984, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.056487798690796, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": 0.0681166723370552, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7571184635162354, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3858171701431274, "metrics/particle/momentum/class_5/event/px_bias": -0.2298370599746704, "metrics/particle/momentum/class_5/event/py_bias": 0.2430282086133957, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8533931374549866, "metrics/particle/momentum/class_5/event/pz_bias": -0.15253838896751404, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.578768014907837, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30225127935409546, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3035811185836792, "metrics/set_slot/angular_recall_all_dr0p10": 0.963505744934082, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8637436628341675, "metrics/set_slot/soft_count_bias_mean": -0.3758014142513275}, "step_train_time": 4.983137607574463, "step_valid_time": 27083.33555817604, "step_total_time": 27088.318696022034, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0091715341868177, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11619835406697165, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3010, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2908, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9606871489924017, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9661129568106313, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9633924134503893, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03388704318936877, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9557317476048893, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9611295681063123, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9584230578101706, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9606871489924017, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046537912660385, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11417787613886332, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2157, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2149, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9535809018567639, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578152753108348, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556933983163491, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04218472468916518, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500442086648984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9542628774422736, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9521488701816571, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9535809018567639, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048252793495172, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10505055240178174, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9433229813664596, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9477379095163807, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9455252918287937, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05226209048361934, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409937888198758, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9453978159126365, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9431906614785992, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9433229813664596} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_39000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..000da98cab48ffcfa9a12fb3475b319a5f8fe203 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.847640704945992, "train": {"Classification_binary": 0.13005991280078888, "Cardinality": 0.11412414163351059, "Classification": 0.062045276165008545, "Regression_pt": 0.09158918261528015, "Regression_eta": 0.0010813606204465032, "Regression_sin_phi": 0.0005702641210518777, "Regression_cos_phi": 0.0006398470140993595, "Regression_energy": 0.09115665405988693, "HitGrouping": 0.13829092681407928, "Auxiliary": 0.1738148033618927, "Total": 0.8108790516853333, "Total_unweighted": 0.37714249739656225}, "train_group": {}, "valid": {"Classification_binary": 0.13118095199267069, "Cardinality": 0.09693090120951335, "Classification": 0.06440949936707814, "Regression_pt": 0.09129719932874043, "Regression_eta": 0.0012070192024111748, "Regression_sin_phi": 0.0007093242214371761, "Regression_cos_phi": 0.0007113396810988585, "Regression_energy": 0.09102489550908406, "HitGrouping": 0.14365516106287637, "Auxiliary": 0.17090908686319986, "Total": 0.8176315625508627, "Total_unweighted": 0.3805402293025205, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8491625785827637, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6115946769714355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2835552990436554, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6177917122840881, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6224486231803894, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3279493749141693, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7968022227287292, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5513204336166382, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3043036162853241, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5594177842140198, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5711278319358826, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41477465629577637, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8607639074325562, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5980114340782166, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27300897240638733, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.6006962060928345, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6031593084335327, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3098853528499603, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8512935638427734, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5981944799423218, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2792097330093384, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6024948954582214, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6064934730529785, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3251063823699951, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026574764400720596, "metrics/hit_grouping/calorimeter/background_recall": 0.6489842534065247, "metrics/hit_grouping/calorimeter/completeness": 0.6717953681945801, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6818109750747681, "metrics/hit_grouping/calorimeter/energy_purity": 0.6740082502365112, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.31778520345687866, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8688305616378784, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6020593643188477, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22502239048480988, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6101450324058533, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.604397177696228, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27423110604286194, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8548259735107422, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6228952407836914, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2859049141407013, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6269897222518921, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6343700885772705, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3292091190814972, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8104925155639648, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5642729997634888, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38692960143089294, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5607762336730957, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5818802714347839, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.42851725220680237, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8405159711837769, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6472226977348328, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32943007349967957, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.653416633605957, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6703906059265137, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3646838963031769, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8302631378173828, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7110540866851807, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4076399803161621, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7071579098701477, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7191874980926514, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.363160640001297, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8655712604522705, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5310441255569458, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20878532528877258, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5343577861785889, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5233791470527649, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27768850326538086, "metrics/hit_grouping/calorimeter/purity": 0.662434995174408, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8166128993034363, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1497807800769806, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0938461571931839, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15063369274139404, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12115941196680069, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03076923079788685, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.810672402381897, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30493003129959106, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11231470853090286, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3044280409812927, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2991553544998169, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2502850592136383, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8543691039085388, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6246193647384644, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2927597165107727, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.629269540309906, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6345147490501404, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3356833755970001, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7482886910438538, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3886730372905731, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.549274742603302, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6026400327682495, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5587268471717834, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8053532838821411, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30398204922676086, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5825311541557312, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6024054288864136, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4167134165763855, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6897243857383728, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.417263925075531, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4726821184158325, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5284526944160461, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6402300000190735, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.717972993850708, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.398686945438385, "metrics/hit_grouping/tracker/all/object_completeness": 0.5061423182487488, "metrics/hit_grouping/tracker/all/object_purity": 0.558129608631134, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5956451892852783, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005694556050002575, "metrics/hit_grouping/tracker/background_recall": 0.054729610681533813, "metrics/hit_grouping/tracker/completeness": 0.5202074646949768, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4186360239982605, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7558279037475586, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3753986358642578, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5809106230735779, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6150926947593689, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.543507993221283, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6953063011169434, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4235674738883972, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4778962731361389, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5467756986618042, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6367790102958679, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.658023476600647, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4239482283592224, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37017831206321716, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4437430202960968, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6676779985427856, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7172822952270508, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4111250936985016, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5071573257446289, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5725818276405334, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.590133011341095, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6866859197616577, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.39280959963798523, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46323445439338684, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5642404556274414, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6384820342063904, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7243741154670715, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38351672887802124, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5123648047447205, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5382277369499207, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5952938199043274, "metrics/hit_grouping/tracker/purity": 0.49624529480934143, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8922760486602783, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05560382455587387, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1554267853498459, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.15579289197921753, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.709077775478363, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28304821252822876, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40205034613609314, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4230796992778778, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5298081636428833, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6953009963035583, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4584735333919525, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5656512379646301, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.628178060054779, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6735730171203613, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5848235487937927, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7220104336738586, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.821336030960083, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6666767597198486, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09909621626138687, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1358720362186432, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.80229377746582, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.31695666909217834, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1865168660879135, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9245944023132324, "metrics/particle/class_1/pid_accuracy": 0.9768880009651184, "metrics/particle/class_2/efficiency": 0.5692169666290283, "metrics/particle/class_2/pid_accuracy": 0.8048876523971558, "metrics/particle/class_3/efficiency": 0.8158668279647827, "metrics/particle/class_3/pid_accuracy": 0.9793320894241333, "metrics/particle/class_4/efficiency": 0.7882579565048218, "metrics/particle/class_4/pid_accuracy": 0.6117216348648071, "metrics/particle/class_5/efficiency": 0.9580000042915344, "metrics/particle/class_5/pid_accuracy": 0.6701461672782898, "metrics/particle/count/bias_mean": -1.32421875, "metrics/particle/count/mae": 3.0872395038604736, "metrics/particle/count/prediction_mean": 68.73177337646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05496220290660858, "metrics/particle/event/energy_response_mean": 0.9781001210212708, "metrics/particle/event/met_abs_error": 4.835878849029541, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04284631833434105, "metrics/particle/event/scalar_pt_response_mean": 0.9885635375976562, "metrics/particle/event/vector_pt_closure": 0.04378430172801018, "metrics/particle/matched/delta_eta_abs_mean": 0.012125157751142979, "metrics/particle/matched/delta_phi_abs_mean": 0.013174694962799549, "metrics/particle/matched/delta_r_mean": 0.02014724351465702, "metrics/particle/matched/energy_relative_abs_mean": 0.12214077264070511, "metrics/particle/matched/pid_accuracy": 0.9634372591972351, "metrics/particle/matched/pt_relative_abs_mean": 0.12136732786893845, "metrics/particle/matching/duplicate_fraction": 0.03984010964632034, "metrics/particle/matching/efficiency": 0.846895158290863, "metrics/particle/matching/f1": 0.8549756407737732, "metrics/particle/matching/purity": 0.8632118105888367, "metrics/particle/matching/target_coverage_dr0p05": 0.8082820773124695, "metrics/particle/matching/target_coverage_dr0p10": 0.8821812868118286, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.694254994392395, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8762243986129761, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02020660974085331, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12105385959148407, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11951418220996857, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016461282968521118, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15882857143878937, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004986990243196487, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5773280262947083, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.012017653323709965, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6162818670272827, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9224.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1812664270401001, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8510770797729492, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7856106758117676, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6541787981987, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09441489726305008, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.804100036621094, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010448328219354153, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8197425007820129, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.038903556764125824, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6253631114959717, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10528.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8434528112411499, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7408034205436707, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.347053527832031, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8960816860198975, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.026965444907546043, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.05917896330356598, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.404112815856934, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5220011472702026, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.88841438293457, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021554088220000267, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13347382843494415, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1299884021282196, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02514965832233429, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1705276519060135, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020455602556467056, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.222787857055664, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0009615324670448899, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3544695377349854, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2151.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.34008273482322693, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9192799925804138, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.902478814125061, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7593755722045898, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.13698112964630127, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.7197265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.047924529761075974, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.041015148162842, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.017378760501742363, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.49537467956543, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2650.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7120158672332764, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.2913906574249268, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.507620811462402, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.088455319404602, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04706839844584465, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1852526217699051, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.378949165344238, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.193773090839386, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.834848403930664, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019721422344446182, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12041790038347244, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12018383294343948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0143187390640378, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15938438475131989, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004842093912884593, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19857434928417206, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00729436194524169, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6382495164871216, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18587.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14785505831241608, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7694541811943054, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5738906860351562, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4379750192165375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08807007223367691, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.476654052734375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002999143209308386, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.32312092185020447, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.00035998839302919805, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2856009006500244, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21006.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8670523166656494, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.2989468574523926, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.792665481567383, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.748699188232422, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.2818989753723145, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.38567623496055603, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.962339162826538, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.06433875858783722, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.168026447296143, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02000235952436924, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12155097723007202, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12068156152963638, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015755897387862206, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16004016995429993, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033041853457689285, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.38870540261268616, "metrics/particle/momentum/all/accepted/pz_bias": -0.0007563320104964077, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3655649423599243, "metrics/particle/momentum/all/accepted/samples": 29962.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17304693162441254, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8062071800231934, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7420905232429504, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6069987416267395, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09381581842899323, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.881096839904785, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008776035159826279, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6092446446418762, "metrics/particle/momentum/all/assigned/pz_bias": 0.013107547536492348, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.3408279418945312, "metrics/particle/momentum/all/assigned/samples": 34184.0, "metrics/particle/momentum/all/efficiency": 0.846527636051178, "metrics/particle/momentum/all/event/energy_bias": -5.331141471862793, "metrics/particle/momentum/all/event/p3_error_mean": 14.676070213317871, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.023467540740967, "metrics/particle/momentum/all/event/px_bias": -1.2078649997711182, "metrics/particle/momentum/all/event/py_bias": 0.2596025764942169, "metrics/particle/momentum/all/event/pz_abs_mean": 11.021458625793457, "metrics/particle/momentum/all/event/pz_bias": -0.6514354348182678, "metrics/particle/momentum/all/event/pz_residual_rms": 18.55759620666504, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021346820518374443, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09478732943534851, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09442403167486191, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010430621914565563, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12836912274360657, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002977539086714387, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.38389039039611816, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.027182800695300102, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2518712282180786, "metrics/particle/momentum/class_1/accepted/samples": 15449.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08514120429754257, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1944507360458374, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.20283031463623047, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.0967632383108139, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03844282403588295, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7405439615249634, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005352797918021679, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.47569540143013, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.025834105908870697, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6519830226898193, "metrics/particle/momentum/class_1/assigned/samples": 16440.0, "metrics/particle/momentum/class_1/efficiency": 0.9258105158805847, "metrics/particle/momentum/class_1/event/energy_bias": 2.6577889919281006, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.551335334777832, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.698286533355713, "metrics/particle/momentum/class_1/event/px_bias": -0.5400323867797852, "metrics/particle/momentum/class_1/event/py_bias": -0.06367456167936325, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.5147066116333, "metrics/particle/momentum/class_1/event/pz_bias": -1.1807570457458496, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.644922256469727, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026738479733467102, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17096510529518127, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17229750752449036, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.00644309539347887, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21074697375297546, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02023809589445591, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4022576808929443, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.005870176944881678, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.543144702911377, "metrics/particle/momentum/class_2/accepted/samples": 1680.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5060290098190308, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5227760672569275, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6460513472557068, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.33888375759124756, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.22839756309986115, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.517451047897339, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04219067096710205, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1625232696533203, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1206381618976593, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.525854587554932, "metrics/particle/momentum/class_2/assigned/samples": 2465.0, "metrics/particle/momentum/class_2/efficiency": 0.5643265247344971, "metrics/particle/momentum/class_2/event/energy_bias": -8.844575881958008, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.048150062561035, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.321267127990723, "metrics/particle/momentum/class_2/event/px_bias": -0.04623095691204071, "metrics/particle/momentum/class_2/event/py_bias": -0.02410704642534256, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.80400276184082, "metrics/particle/momentum/class_2/event/pz_bias": -0.7785240411758423, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.11693572998047, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017404906451702118, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1476496458053589, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14592356979846954, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02318931557238102, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18506468832492828, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010490638669580221, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23680636286735535, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.028538178652524948, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8116152286529541, "metrics/particle/momentum/class_3/accepted/samples": 12392.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21458594501018524, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4913311004638672, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3389627933502197, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1998136043548584, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13247545063495636, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.22518253326416, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006637318059802055, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4782218933105469, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04053506627678871, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.0357353687286377, "metrics/particle/momentum/class_3/assigned/samples": 14765.0, "metrics/particle/momentum/class_3/efficiency": 0.8148878812789917, "metrics/particle/momentum/class_3/event/energy_bias": 1.5022691488265991, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.211991310119629, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.362656593322754, "metrics/particle/momentum/class_3/event/px_bias": -0.2832258939743042, "metrics/particle/momentum/class_3/event/py_bias": 0.20236244797706604, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.018002033233643, "metrics/particle/momentum/class_3/event/pz_bias": 0.7789815068244934, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.798079490661621, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02381918393075466, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1417548656463623, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13719871640205383, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03061465173959732, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18155989050865173, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013888888992369175, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7806466221809387, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3041676878929138, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2338128089904785, "metrics/particle/momentum/class_4/accepted/samples": 288.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2637343406677246, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.8783621788024902, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.800852656364441, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6786829233169556, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1436619758605957, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.694046974182129, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019718309864401817, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1116060018539429, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.03777251020073891, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.7006688117980957, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.791208803653717, "metrics/particle/momentum/class_4/event/energy_bias": -0.5801775455474854, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8046765327453613, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8569283485412598, "metrics/particle/momentum/class_4/event/px_bias": -0.0887613445520401, "metrics/particle/momentum/class_4/event/py_bias": 0.0028958916664123535, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8046565055847168, "metrics/particle/momentum/class_4/event/pz_bias": 0.508876621723175, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.4855170249938965, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013473741710186005, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1295417994260788, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1297115832567215, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02569996938109398, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1678682118654251, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.013071895577013493, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3107562065124512, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3517740070819855, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.5183424949645996, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04003586992621422, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20534880459308624, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.19805273413658142, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07753238826990128, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03144654259085655, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7186571359634399, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.01886792480945587, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.3823692798614502, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.22956863045692444, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.6597321033477783, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": -0.06644521653652191, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6217677593231201, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2946845293045044, "metrics/particle/momentum/class_5/event/px_bias": -0.24961453676223755, "metrics/particle/momentum/class_5/event/py_bias": 0.14212584495544434, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7601116299629211, "metrics/particle/momentum/class_5/event/pz_bias": 0.01998748630285263, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.3656227588653564, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.29899483919143677, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.30012425780296326, "metrics/set_slot/angular_recall_all_dr0p10": 0.9628180861473083, "metrics/set_slot/angular_recall_selected_dr0p10": 0.867749035358429, "metrics/set_slot/soft_count_bias_mean": -0.34590718150138855}, "step_train_time": 5.1329522132873535, "step_valid_time": 30188.494626522064, "step_total_time": 30193.62757897377, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976812102670375, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10496497394846926, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9580442682523951, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9605829744948658, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.959311941779689, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03941702550513415, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534192269573836, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9559456773766147, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546807806814422, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9580442682523951, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.994289519251455, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10587772133815676, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2157, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9535809018567639, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9608017817371938, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571777235411582, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039198218262806235, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526967285587975, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9599109131403119, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9562902152207677, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9535809018567639, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9927756145139975, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10099952293129055, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355590062111802, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.94140625, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9384735202492211, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05859375, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9324534161490683, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.93828125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9353582554517134, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355590062111802} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_4000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..07ce4e7064eb42051570754be7720b7f46cb691c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.18950161076369149, "train": {"Classification_binary": 0.3428344130516052, "Cardinality": 0.21685941517353058, "Classification": 0.2160073071718216, "Regression_pt": 0.3854466676712036, "Regression_eta": 0.004641529638320208, "Regression_sin_phi": 0.002547112526372075, "Regression_cos_phi": 0.0025538867339491844, "Regression_energy": 0.3812277913093567, "HitGrouping": 0.24549011886119843, "Auxiliary": 0.4280570149421692, "Total": 2.279686689376831, "Total_unweighted": 1.3352587081026286}, "train_group": {}, "valid": {"Classification_binary": 0.3514440457026164, "Cardinality": 0.21586235364278158, "Classification": 0.2363139589627584, "Regression_pt": 0.3737014929453532, "Regression_eta": 0.005167270389695962, "Regression_sin_phi": 0.002751430496573448, "Regression_cos_phi": 0.0027449289336800575, "Regression_energy": 0.3690714438756307, "HitGrouping": 0.2453288435935974, "Auxiliary": 0.43523216247558594, "Total": 2.3380114237467446, "Total_unweighted": 1.341194571306308, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.799767255783081, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2531580328941345, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35383015871047974, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.2513594627380371, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.23251183331012726, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.47832247614860535, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7609972357749939, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2616175413131714, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.36563876271247864, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.25920817255973816, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2343389242887497, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.523551344871521, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8097329139709473, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.26648014783859253, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35079020261764526, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2654912769794464, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2518311142921448, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.45739075541496277, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8021546006202698, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2617570757865906, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3531002998352051, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.26038017868995667, "metrics/hit_grouping/calorimeter/all/object_purity": 0.24404531717300415, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.470060795545578, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025424614548683167, "metrics/hit_grouping/calorimeter/background_recall": 0.6250917911529541, "metrics/hit_grouping/calorimeter/completeness": 0.3685092628002167, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3932555615901947, "metrics/hit_grouping/calorimeter/energy_purity": 0.388683557510376, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5440686345100403, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8334679007530212, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.25535866618156433, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27769482135772705, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2555394768714905, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.23557108640670776, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4005971848964691, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7881565690040588, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2872854471206665, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38379427790641785, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.28463560342788696, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.27587151527404785, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5046094059944153, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.750461757183075, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24869737029075623, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4807230532169342, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2453499585390091, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.22881671786308289, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5810896158218384, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7643107175827026, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2926957607269287, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4375583827495575, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.29210489988327026, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2823250889778137, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.557022750377655, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7325531244277954, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4619666337966919, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6520146727561951, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4554581940174103, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.44038280844688416, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5829408764839172, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.85174560546875, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2028765082359314, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22405250370502472, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.20131126046180725, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.17776821553707123, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3621266782283783, "metrics/hit_grouping/calorimeter/purity": 0.3629458546638489, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8991214632987976, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04306633397936821, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10769230872392654, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04228205233812332, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.008711201138794422, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04153846204280853, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.875037431716919, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.0630372166633606, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09407069534063339, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.063072070479393, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.04060760885477066, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23318129777908325, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7958958745002747, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2778478264808655, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3732292652130127, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2763770818710327, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.26076218485832214, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4928187429904938, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6548261642456055, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4237895905971527, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.28288090229034424, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.278352826833725, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7214584350585938, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7359896302223206, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.4183959662914276, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.35300523042678833, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3029683828353882, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5687044262886047, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5888715386390686, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4173426926136017, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.23091095685958862, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2301885336637497, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7827833294868469, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6221079230308533, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41946911811828613, "metrics/hit_grouping/tracker/all/object_completeness": 0.2576196789741516, "metrics/hit_grouping/tracker/all/object_purity": 0.2515413463115692, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7453712224960327, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006655340548604727, "metrics/hit_grouping/tracker/background_recall": 0.03411715850234032, "metrics/hit_grouping/tracker/completeness": 0.27782708406448364, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5860953330993652, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6211629509925842, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41904327273368835, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.30331534147262573, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.30415624380111694, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7567198276519775, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.605276346206665, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41785645484924316, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.22143197059631348, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.2252790778875351, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7645155191421509, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.642089307308197, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4221278429031372, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1946309357881546, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.16264718770980835, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6998381614685059, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6371757984161377, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44850826263427734, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2578960359096527, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2481105476617813, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.73157799243927, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6878935098648071, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5033289194107056, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.33127838373184204, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.30542656779289246, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6551265120506287, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5909108519554138, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3668929934501648, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2443077117204666, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2465357929468155, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7792086005210876, "metrics/hit_grouping/tracker/purity": 0.2652859389781952, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8946498036384583, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018679408356547356, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.023569317534565926, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.01845608279109001, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07862728089094162, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6594607830047607, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.15033696591854095, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.09610641747713089, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.08174557238817215, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6039398908615112, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5808206796646118, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.504574716091156, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3078140616416931, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3025476634502411, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8520748615264893, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.460954487323761, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.3535241186618805, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.3636248707771301, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5411188006401062, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.02477286383509636, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.010314030572772026, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.093975067138672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.18984542787075043, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.059405602514743805, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5422239899635315, "metrics/particle/class_1/pid_accuracy": 0.9014377593994141, "metrics/particle/class_2/efficiency": 0.36078080534935, "metrics/particle/class_2/pid_accuracy": 0.2590174078941345, "metrics/particle/class_3/efficiency": 0.4523630738258362, "metrics/particle/class_3/pid_accuracy": 0.8807265758514404, "metrics/particle/class_4/efficiency": 0.48026949167251587, "metrics/particle/class_4/pid_accuracy": 0.24849699437618256, "metrics/particle/class_5/efficiency": 0.47600001096725464, "metrics/particle/class_5/pid_accuracy": 0.21008403599262238, "metrics/particle/count/bias_mean": -9.59765625, "metrics/particle/count/mae": 10.243489265441895, "metrics/particle/count/prediction_mean": 60.45833206176758, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.13157938420772552, "metrics/particle/event/energy_response_mean": 0.9814531803131104, "metrics/particle/event/met_abs_error": 13.718268394470215, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11350973695516586, "metrics/particle/event/scalar_pt_response_mean": 1.009170413017273, "metrics/particle/event/vector_pt_closure": 0.10547897219657898, "metrics/particle/matched/delta_eta_abs_mean": 0.022339288145303726, "metrics/particle/matched/delta_phi_abs_mean": 0.023673120886087418, "metrics/particle/matched/delta_r_mean": 0.03626101091504097, "metrics/particle/matched/energy_relative_abs_mean": 0.19965828955173492, "metrics/particle/matched/pid_accuracy": 0.8443643450737, "metrics/particle/matched/pt_relative_abs_mean": 0.19756558537483215, "metrics/particle/matching/duplicate_fraction": 0.12207098305225372, "metrics/particle/matching/efficiency": 0.4873613119125366, "metrics/particle/matching/f1": 0.5232004523277283, "metrics/particle/matching/purity": 0.5647290945053101, "metrics/particle/matching/target_coverage_dr0p05": 0.4119472801685333, "metrics/particle/matching/target_coverage_dr0p10": 0.5758043527603149, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.302622526884079, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5598386526107788, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.035441722720861435, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19963552057743073, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.198023721575737, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.017303355038166046, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23957118391990662, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.016798218712210655, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.362592101097107, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.08676876872777939, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.10544490814209, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4941.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6351812481880188, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.6572346687316895, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.1954140663146973, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.9925596714019775, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3459944725036621, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.8084716796875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03430018573999405, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.074201822280884, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.049881573766469955, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.85408353805542, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8688.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4518105387687683, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.1898329257965088, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.742034912109375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.341432571411133, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.0605618953704834, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.25887331366539, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.448003768920898, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -6.250697135925293, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 23.513721466064453, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.034147147089242935, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.2077668160200119, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.20402903854846954, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.010941377840936184, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.244187131524086, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04651162773370743, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.4755547046661377, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08141744881868362, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.6204328536987305, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1075.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8267601132392883, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8247827291488647, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.632822513580322, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.4435553550720215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3623492419719696, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.92859649658203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07760880887508392, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.363651990890503, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.22654154896736145, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.41891622543335, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1907.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3558424413204193, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.023147583007812, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 15.407142639160156, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.1191296577453613, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17185412347316742, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.4395137429237366, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 15.168889045715332, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.7893686294555664, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 32.83346176147461, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03681890666484833, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19876869022846222, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19707787036895752, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.01894468627870083, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23742026090621948, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0020323407370597124, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.449351042509079, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0017491289181634784, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1920441389083862, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11317.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5709893107414246, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.4200704097747803, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.854929208755493, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.655224561691284, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.34646928310394287, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.265091896057129, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0135527104139328, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0176180601119995, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05114969611167908, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.0851259231567383, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19996.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5279189944267273, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 9.297525405883789, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.643857955932617, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.574906349182129, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.9833894968032837, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3311942219734192, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.264535903930664, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3201366066932678, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 14.507672309875488, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03626061975955963, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19957385957241058, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1977786272764206, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.017980434000492096, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23845957219600677, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.009000172838568687, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.83534836769104, "metrics/particle/momentum/all/accepted/pz_bias": -0.028642093762755394, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.374035120010376, "metrics/particle/momentum/all/accepted/samples": 17333.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6051644682884216, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.103973150253296, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.06246018409729, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.862511157989502, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3473243713378906, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.27385425567627, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.023438265547156334, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4639416933059692, "metrics/particle/momentum/all/assigned/pz_bias": -0.061723221093416214, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.042820930480957, "metrics/particle/momentum/all/assigned/samples": 30591.0, "metrics/particle/momentum/all/efficiency": 0.4897157847881317, "metrics/particle/momentum/all/event/energy_bias": -0.535788893699646, "metrics/particle/momentum/all/event/p3_error_mean": 37.431480407714844, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.706024169921875, "metrics/particle/momentum/all/event/px_bias": 1.8720974922180176, "metrics/particle/momentum/all/event/py_bias": -0.5118347406387329, "metrics/particle/momentum/all/event/pz_abs_mean": 28.725990295410156, "metrics/particle/momentum/all/event/pz_bias": -9.719929695129395, "metrics/particle/momentum/all/event/pz_residual_rms": 41.476043701171875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.043323758989572525, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2170625925064087, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21435986459255219, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03789324313402176, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.25353553891181946, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.010116560384631157, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9600108861923218, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04677092283964157, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3487565517425537, "metrics/particle/momentum/class_1/accepted/samples": 9094.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.4444744884967804, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.5752949714660645, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.7102605104446411, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.4766148328781128, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.30044904351234436, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.111968994140625, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.020105890929698944, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.389523983001709, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.08347322046756744, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.535024642944336, "metrics/particle/momentum/class_1/assigned/samples": 14921.0, "metrics/particle/momentum/class_1/efficiency": 0.5449751019477844, "metrics/particle/momentum/class_1/event/energy_bias": 27.737918853759766, "metrics/particle/momentum/class_1/event/p3_error_mean": 29.149600982666016, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.87468147277832, "metrics/particle/momentum/class_1/event/px_bias": 3.1811559200286865, "metrics/particle/momentum/class_1/event/py_bias": -0.10698824375867844, "metrics/particle/momentum/class_1/event/pz_abs_mean": 21.23908042907715, "metrics/particle/momentum/class_1/event/pz_bias": -8.781689643859863, "metrics/particle/momentum/class_1/event/pz_residual_rms": 31.0821590423584, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04159404709935188, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23200398683547974, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22562594711780548, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08743759244680405, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26711153984069824, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0295475535094738, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8603836297988892, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13851821422576904, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.663764953613281, "metrics/particle/momentum/class_2/accepted/samples": 1083.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6617339849472046, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7348124384880066, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9921472668647766, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.604594886302948, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3237623870372772, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.929748296737671, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05445544421672821, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.71714448928833, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1400350034236908, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.597836971282959, "metrics/particle/momentum/class_2/assigned/samples": 2020.0, "metrics/particle/momentum/class_2/efficiency": 0.36378905177116394, "metrics/particle/momentum/class_2/event/energy_bias": -23.45516586303711, "metrics/particle/momentum/class_2/event/p3_error_mean": 26.8016357421875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.989009857177734, "metrics/particle/momentum/class_2/event/px_bias": -1.0350263118743896, "metrics/particle/momentum/class_2/event/py_bias": -0.9442319273948669, "metrics/particle/momentum/class_2/event/pz_abs_mean": 20.154895782470703, "metrics/particle/momentum/class_2/event/pz_bias": -0.008126884698867798, "metrics/particle/momentum/class_2/event/pz_residual_rms": 32.12307357788086, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02602916583418846, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17112509906291962, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1712268888950348, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019012853503227234, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21135638654232025, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.004056795034557581, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4830087125301361, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0014654898550361395, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.7720295190811157, "metrics/particle/momentum/class_3/accepted/samples": 6902.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7797304391860962, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.168152332305908, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.915865421295166, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.7847442626953125, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4041941165924072, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.45535659790039, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02157619781792164, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.3145430088043213, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.09185946732759476, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.9270148277282715, "metrics/particle/momentum/class_3/assigned/samples": 13209.0, "metrics/particle/momentum/class_3/efficiency": 0.45386993885040283, "metrics/particle/momentum/class_3/event/energy_bias": 0.9515678286552429, "metrics/particle/momentum/class_3/event/p3_error_mean": 18.240585327148438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.170308113098145, "metrics/particle/momentum/class_3/event/px_bias": 0.22418582439422607, "metrics/particle/momentum/class_3/event/py_bias": -0.4038809835910797, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.939576148986816, "metrics/particle/momentum/class_3/event/pz_bias": -1.6098953485488892, "metrics/particle/momentum/class_3/event/pz_residual_rms": 23.02906036376953, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.037766970694065094, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20240403711795807, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19548825919628143, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028028834611177444, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2344813048839569, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011428571306169033, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.654420018196106, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4189762771129608, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.204946756362915, "metrics/particle/momentum/class_4/accepted/samples": 175.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.655197262763977, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.1256937980651855, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.664997100830078, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.481802463531494, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36335402727127075, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.551688194274902, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.021739130839705467, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.1988368034362793, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.6538171172142029, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 5.96553897857666, "metrics/particle/momentum/class_4/assigned/samples": 322.0, "metrics/particle/momentum/class_4/efficiency": 0.48076921701431274, "metrics/particle/momentum/class_4/event/energy_bias": -1.5154671669006348, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.1920623779296875, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.9841904640197754, "metrics/particle/momentum/class_4/event/px_bias": 0.23436734080314636, "metrics/particle/momentum/class_4/event/py_bias": 0.16213740408420563, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4631776809692383, "metrics/particle/momentum/class_4/event/pz_bias": 0.9682326912879944, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.688590049743652, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.040635038167238235, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2210100144147873, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2321140021085739, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.10747578740119934, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.27156171202659607, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4013965129852295, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.057455334812402725, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.5021233558654785, "metrics/particle/momentum/class_5/accepted/samples": 79.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.2810920178890228, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.40120768547058105, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3948400020599365, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.08059664815664291, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2689075767993927, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5675038695335388, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1260504275560379, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.116792678833008, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.649584949016571, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.365669250488281, "metrics/particle/momentum/class_5/assigned/samples": 119.0, "metrics/particle/momentum/class_5/efficiency": 0.49685534834861755, "metrics/particle/momentum/class_5/event/energy_bias": -4.254642963409424, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.614443778991699, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.5660486221313477, "metrics/particle/momentum/class_5/event/px_bias": -0.7325856685638428, "metrics/particle/momentum/class_5/event/py_bias": 0.7811292409896851, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.39274525642395, "metrics/particle/momentum/class_5/event/pz_bias": -0.28845009207725525, "metrics/particle/momentum/class_5/event/pz_residual_rms": 7.797005653381348, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6733292937278748, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6771455407142639, "metrics/set_slot/angular_recall_all_dr0p10": 0.8245357871055603, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6034235954284668, "metrics/set_slot/soft_count_bias_mean": -0.044419605284929276}, "step_train_time": 5.294837236404419, "step_valid_time": 12008.262516260147, "step_total_time": 12013.557353973389, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0823353249448533, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28041159209294364, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2885, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2526, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2362, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8344895936570862, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8755632582322357, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.854533152909337, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1244367417677643, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7803105384869508, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8187175043327556, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7990527740189445, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8344895936570862, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0451765104283368, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26086119847855194, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2217, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1925, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1866, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8510167992926614, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8682904826341904, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.859566867604376, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13170951736580966, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8249336870026526, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8416779431664412, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8332217012726055, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8510167992926614, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.034572143743813, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27999045257505917, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1229, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1043, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1006, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8097826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8486574450772986, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8287644020659515, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15134255492270138, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.781055900621118, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8185516680227828, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7993643226062773, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8097826086956522} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_40000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..d13bd391ea0d591a9d4953d3a568d3134f5dac18 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.895016107636915, "train": {"Classification_binary": 0.14906282722949982, "Cardinality": 0.1215101033449173, "Classification": 0.06008356437087059, "Regression_pt": 0.09657294303178787, "Regression_eta": 0.001203068532049656, "Regression_sin_phi": 0.0005988741177134216, "Regression_cos_phi": 0.0006848928751423955, "Regression_energy": 0.09624908864498138, "HitGrouping": 0.14309491217136383, "Auxiliary": 0.18208125233650208, "Total": 0.8634916543960571, "Total_unweighted": 0.40445525880204514}, "train_group": {}, "valid": {"Classification_binary": 0.13489281137784323, "Cardinality": 0.096719096104304, "Classification": 0.06654611726601918, "Regression_pt": 0.0923598309357961, "Regression_eta": 0.001165266614407301, "Regression_sin_phi": 0.0006748675368726254, "Regression_cos_phi": 0.0006724434594313303, "Regression_energy": 0.09156038363774617, "HitGrouping": 0.1442921757698059, "Auxiliary": 0.1729539434115092, "Total": 0.819266160329183, "Total_unweighted": 0.38787172082811594, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8489344716072083, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6105115413665771, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2851610481739044, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6159689426422119, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6192500591278076, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.333616703748703, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7968816757202148, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5513763427734375, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3012538254261017, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5584908127784729, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5742138028144836, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4320569336414337, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8593798279762268, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5895965695381165, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2640739679336548, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5927218198776245, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5994597673416138, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31628963351249695, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8504126667976379, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5928986668586731, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27419453859329224, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.59713214635849, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6035637259483337, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.332249253988266, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026447243988513947, "metrics/hit_grouping/calorimeter/background_recall": 0.64734947681427, "metrics/hit_grouping/calorimeter/completeness": 0.6675593256950378, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6793336272239685, "metrics/hit_grouping/calorimeter/energy_purity": 0.6715410947799683, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3137281537055969, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8698835372924805, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5973522067070007, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.215825617313385, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6043446063995361, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6023169755935669, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2799641788005829, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8513173460960388, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6186281442642212, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2851771116256714, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6230704188346863, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.631130576133728, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.34036874771118164, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8082564473152161, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5566592812538147, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3862975537776947, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5548359155654907, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5774752497673035, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43445834517478943, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.83949214220047, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6402279734611511, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3236374855041504, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6470293998718262, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6667924523353577, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.37010276317596436, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8282220959663391, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7078896164894104, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.42019885778427124, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7046067118644714, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.715436577796936, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3663003742694855, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8649925589561462, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5273019075393677, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20235703885555267, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5297367572784424, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5212730765342712, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2871969938278198, "metrics/hit_grouping/calorimeter/purity": 0.6581717133522034, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.822083055973053, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.16103166341781616, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10307691991329193, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15917764604091644, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12591561675071716, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8212257027626038, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2923123836517334, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0946408212184906, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.29121652245521545, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28827354311943054, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24230331182479858, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8526952266693115, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.619391918182373, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28816893696784973, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6240619421005249, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6318786144256592, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34371721744537354, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7546624541282654, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.391661673784256, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5510560274124146, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5904387831687927, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5449790954589844, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8119823932647705, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.29332584142684937, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5833713412284851, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.594839870929718, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4026920795440674, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6894492506980896, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4173426926136017, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.47182342410087585, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5194910168647766, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6396786570549011, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7203808426856995, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.398781418800354, "metrics/hit_grouping/tracker/all/object_completeness": 0.5062611103057861, "metrics/hit_grouping/tracker/all/object_purity": 0.5482615232467651, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5897883772850037, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004087147768586874, "metrics/hit_grouping/tracker/background_recall": 0.03565717115998268, "metrics/hit_grouping/tracker/completeness": 0.5180915594100952, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4275165796279907, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7546355128288269, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37849658727645874, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5786728262901306, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6062596440315247, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5445102453231812, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6993575692176819, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41538169980049133, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.48055562376976013, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5365591049194336, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6261184215545654, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6666773557662964, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4261731505393982, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37282902002334595, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4319474697113037, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6516990065574646, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7218791246414185, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40564343333244324, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5092649459838867, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.564284086227417, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5799784064292908, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7045124173164368, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41877496242523193, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4814012944698334, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5468305349349976, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.610519289970398, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7212192416191101, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3863263726234436, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.506719172000885, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5276399850845337, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5989229679107666, "metrics/hit_grouping/tracker/purity": 0.49342912435531616, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8944621086120605, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.06168549135327339, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16126343607902527, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1523796021938324, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.076455257833004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.708417534828186, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2643857002258301, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.388992041349411, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.41886165738105774, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5194401144981384, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6980883479118347, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.459890216588974, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5664934515953064, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6167895793914795, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.66755211353302, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5897571444511414, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7319031357765198, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8324874639511108, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6545697450637817, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10402979701757431, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12376496195793152, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.757471084594727, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3153770864009857, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18493729829788208, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9272655248641968, "metrics/particle/class_1/pid_accuracy": 0.9743512868881226, "metrics/particle/class_2/efficiency": 0.5673098564147949, "metrics/particle/class_2/pid_accuracy": 0.8028475642204285, "metrics/particle/class_3/efficiency": 0.8170896768569946, "metrics/particle/class_3/pid_accuracy": 0.9808333516120911, "metrics/particle/class_4/efficiency": 0.7767083644866943, "metrics/particle/class_4/pid_accuracy": 0.6146220564842224, "metrics/particle/class_5/efficiency": 0.9520000219345093, "metrics/particle/class_5/pid_accuracy": 0.6470588445663452, "metrics/particle/count/bias_mean": -0.751953125, "metrics/particle/count/mae": 3.0006511211395264, "metrics/particle/count/prediction_mean": 69.30403900146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05872057378292084, "metrics/particle/event/energy_response_mean": 1.004500389099121, "metrics/particle/event/met_abs_error": 5.615278244018555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047912564128637314, "metrics/particle/event/scalar_pt_response_mean": 1.0153852701187134, "metrics/particle/event/vector_pt_closure": 0.04703177884221077, "metrics/particle/matched/delta_eta_abs_mean": 0.012708905152976513, "metrics/particle/matched/delta_phi_abs_mean": 0.013141428120434284, "metrics/particle/matched/delta_r_mean": 0.02056545950472355, "metrics/particle/matched/energy_relative_abs_mean": 0.12315265089273453, "metrics/particle/matched/pid_accuracy": 0.9626688361167908, "metrics/particle/matched/pt_relative_abs_mean": 0.123810313642025, "metrics/particle/matching/duplicate_fraction": 0.04787179082632065, "metrics/particle/matching/efficiency": 0.8483820557594299, "metrics/particle/matching/f1": 0.8529597520828247, "metrics/particle/matching/purity": 0.8575870394706726, "metrics/particle/matching/target_coverage_dr0p05": 0.8074828386306763, "metrics/particle/matching/target_coverage_dr0p10": 0.8846067786216736, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6882144212722778, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8773395419120789, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020821349695324898, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.122976154088974, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12356951087713242, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01818813569843769, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16303272545337677, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004575661849230528, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5776171088218689, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.06222013011574745, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5597822666168213, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9179.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20221783220767975, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0836026668548584, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0800433158874512, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9490281343460083, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10101105272769928, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.944681167602539, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01237834244966507, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8664757013320923, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08354765921831131, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5451531410217285, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10583.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.839337944984436, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8566282987594604, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.838932037353516, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0145416259765625, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2364085167646408, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09313622117042542, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.888309955596924, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.40129816532135, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.792909622192383, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021479422226548195, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13256187736988068, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13412214815616608, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023744944483041763, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17597617208957672, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02775208093225956, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3163539171218872, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07224617898464203, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4459218978881836, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2162.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3501947224140167, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5499180555343628, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.672529935836792, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.5342931747436523, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.14200074970722198, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 45.782310485839844, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05320344865322113, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1781668663024902, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1935196816921234, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.767752170562744, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2669.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7156570553779602, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.3147778511047363, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.496252536773682, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.091928243637085, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10639968514442444, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.021641649305820465, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.36560583114624, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.12622076272964478, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.924610137939453, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020020170137286186, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12061046808958054, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12117946892976761, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021961666643619537, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16038434207439423, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005386770353652537, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20139111578464508, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014521835371851921, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6598874926567078, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18564.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15191827714443207, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9106091260910034, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7509813904762268, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6268186569213867, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0936465933918953, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.462546348571777, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034036117140203714, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.35052812099456787, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.008469843305647373, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.704833984375, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21154.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8659793734550476, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.5205368995666504, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.230360507965088, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.114852428436279, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.231806218624115, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7035906910896301, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.209059953689575, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5236861109733582, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.564086437225342, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020371582359075546, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1222006231546402, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1228487566113472, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02093234658241272, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1623736172914505, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003745193127542734, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39747631549835205, "metrics/particle/momentum/all/accepted/pz_bias": 0.033335503190755844, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3695021867752075, "metrics/particle/momentum/all/accepted/samples": 29905.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18277102708816528, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0134140253067017, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.001259684562683, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8738974928855896, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09966284781694412, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.776182174682617, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010027321055531502, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6510059833526611, "metrics/particle/momentum/all/assigned/pz_bias": 0.04591815173625946, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5219340324401855, "metrics/particle/momentum/all/assigned/samples": 34406.0, "metrics/particle/momentum/all/efficiency": 0.8449172377586365, "metrics/particle/momentum/all/event/energy_bias": 2.062387704849243, "metrics/particle/momentum/all/event/p3_error_mean": 15.917387008666992, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.457411766052246, "metrics/particle/momentum/all/event/px_bias": 0.36181509494781494, "metrics/particle/momentum/all/event/py_bias": -0.6320960521697998, "metrics/particle/momentum/all/event/pz_abs_mean": 12.091476440429688, "metrics/particle/momentum/all/event/pz_bias": 2.0512051582336426, "metrics/particle/momentum/all/event/pz_residual_rms": 20.802608489990234, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021515410393476486, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09671120345592499, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09807984530925751, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01861094869673252, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.13227178156375885, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0031735750380903482, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39102914929389954, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.04996326565742493, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.257292628288269, "metrics/particle/momentum/class_1/accepted/samples": 15440.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09075049310922623, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20606893301010132, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21811027824878693, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11954797059297562, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.041707079857587814, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9819623231887817, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005637730471789837, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5028888583183289, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06078962981700897, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.801911473274231, "metrics/particle/momentum/class_1/assigned/samples": 16496.0, "metrics/particle/momentum/class_1/efficiency": 0.9252711534500122, "metrics/particle/momentum/class_1/event/energy_bias": 3.924567699432373, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.130468368530273, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.731389045715332, "metrics/particle/momentum/class_1/event/px_bias": -0.24596118927001953, "metrics/particle/momentum/class_1/event/py_bias": -1.1190087795257568, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.038752555847168, "metrics/particle/momentum/class_1/event/pz_bias": 1.8752877712249756, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.497678756713867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027026169002056122, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17449495196342468, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17562195658683777, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.022067992016673088, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21599219739437103, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022289156913757324, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.420632243156433, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04824482649564743, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.518159866333008, "metrics/particle/momentum/class_2/accepted/samples": 1660.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5421103239059448, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8057947754859924, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8645376563072205, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5865888595581055, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25059008598327637, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.190053462982178, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04838709533214569, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3291120529174805, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13164755702018738, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.859668731689453, "metrics/particle/momentum/class_2/assigned/samples": 2542.0, "metrics/particle/momentum/class_2/efficiency": 0.5576083064079285, "metrics/particle/momentum/class_2/event/energy_bias": -5.671689987182617, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.401518821716309, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.418435573577881, "metrics/particle/momentum/class_2/event/px_bias": 0.5267025232315063, "metrics/particle/momentum/class_2/event/py_bias": 0.33378928899765015, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.199531555175781, "metrics/particle/momentum/class_2/event/pz_bias": -0.7842463254928589, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.268556594848633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018055465072393417, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14639782905578613, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14612075686454773, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02258235216140747, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18567773699760437, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016172070754691958, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24504466354846954, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007516923360526562, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8358127474784851, "metrics/particle/momentum/class_3/accepted/samples": 12367.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22239112854003906, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8979172706604004, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8531851768493652, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7190818786621094, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13754796981811523, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.22008514404297, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008011849597096443, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5071841478347778, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013174462132155895, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2018802165985107, "metrics/particle/momentum/class_3/assigned/samples": 14853.0, "metrics/particle/momentum/class_3/efficiency": 0.8132439255714417, "metrics/particle/momentum/class_3/event/energy_bias": 3.8802943229675293, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.562636375427246, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.4033355712890625, "metrics/particle/momentum/class_3/event/px_bias": 0.10992789268493652, "metrics/particle/momentum/class_3/event/py_bias": -0.015128090977668762, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.310057640075684, "metrics/particle/momentum/class_3/event/pz_bias": 0.6999440789222717, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.703088760375977, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023159610107541084, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14434590935707092, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14222554862499237, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05956244841217995, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18716400861740112, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010526316240429878, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7582506537437439, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13508884608745575, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.96226966381073, "metrics/particle/momentum/class_4/accepted/samples": 285.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2854941785335541, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.3861899375915527, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.1025171279907227, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9866831302642822, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15730337798595428, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.85101890563965, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016853932291269302, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0653142929077148, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.020372873172163963, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4601078033447266, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.7829670310020447, "metrics/particle/momentum/class_4/event/energy_bias": -0.03997894376516342, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.644843339920044, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.706950306892395, "metrics/particle/momentum/class_4/event/px_bias": 0.0934988483786583, "metrics/particle/momentum/class_4/event/py_bias": 0.05700959637761116, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7214034795761108, "metrics/particle/momentum/class_4/event/pz_bias": 0.18486350774765015, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.809353351593018, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014760327525436878, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12997668981552124, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13266202807426453, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03754729405045509, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1718987375497818, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5962096452713013, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09095995128154755, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.021982192993164, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05373380705714226, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15490493178367615, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.15020591020584106, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04630081728100777, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03144654259085655, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.21114447712898254, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025157231837511063, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6968320608139038, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.24838417768478394, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.2811057567596436, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": -0.03080536425113678, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6294482946395874, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.243213176727295, "metrics/particle/momentum/class_5/event/px_bias": -0.1223529800772667, "metrics/particle/momentum/class_5/event/py_bias": 0.11124184727668762, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8256244659423828, "metrics/particle/momentum/class_5/event/pz_bias": 0.0753558948636055, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.5907084941864014, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2996377944946289, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3009829521179199, "metrics/set_slot/angular_recall_all_dr0p10": 0.9635800719261169, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8683995604515076, "metrics/set_slot/soft_count_bias_mean": 0.33457374572753906}, "step_train_time": 5.61181116104126, "step_valid_time": 33385.109914541245, "step_total_time": 33390.721725940704, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016708534124008, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11106100088048898, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2878, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9567228278823918, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9583057577763071, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9575136386179534, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04169424222369292, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507763462173769, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523494374586366, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9515622416928418, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9567228278823918, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0139961568887015, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11211878869871628, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2256, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2141, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9478337754199824, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950354609929078, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9490925188136343, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04964539007092199, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9465075154730327, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9490248226950354, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9477644975652944, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9478337754199824, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0145277251055138, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11044450627115743, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1206, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.94453125, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9415887850467289, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05546875, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9363354037267081, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9421875, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9392523364485982, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386645962732919} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_41000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..01d5a568552d06fb58adb4d10ab5580cafff6c7e --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9423915103278377, "train": {"Classification_binary": 0.13813653588294983, "Cardinality": 0.08749114722013474, "Classification": 0.05901136249303818, "Regression_pt": 0.08739214390516281, "Regression_eta": 0.0009460535366088152, "Regression_sin_phi": 0.000586903712246567, "Regression_cos_phi": 0.0006128702661953866, "Regression_energy": 0.08628945052623749, "HitGrouping": 0.13799089193344116, "Auxiliary": 0.17205053567886353, "Total": 0.7790043354034424, "Total_unweighted": 0.3729753203224391}, "train_group": {}, "valid": {"Classification_binary": 0.12782158454259238, "Cardinality": 0.09578826030095418, "Classification": 0.06435898939768474, "Regression_pt": 0.08971675237019856, "Regression_eta": 0.0011434114227692287, "Regression_sin_phi": 0.0006692536020030578, "Regression_cos_phi": 0.0006716963835060596, "Regression_energy": 0.08911493420600891, "HitGrouping": 0.14136328299840292, "Auxiliary": 0.1692550778388977, "Total": 0.7995545864105225, "Total_unweighted": 0.37349662192476296, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8505010008811951, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.616481602191925, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28081610798835754, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6216646432876587, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6261342763900757, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3270048201084137, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7982253432273865, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5643426775932312, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.29041004180908203, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5732782483100891, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5887821912765503, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.40901389718055725, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.859883725643158, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5955091714859009, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.26138827204704285, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5992711186408997, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6077191233634949, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3107633590698242, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8513338565826416, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5994624495506287, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27024316787719727, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6041457056999207, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6119464039802551, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.324802428483963, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02658986859023571, "metrics/hit_grouping/calorimeter/background_recall": 0.6491508483886719, "metrics/hit_grouping/calorimeter/completeness": 0.6776390075683594, "metrics/hit_grouping/calorimeter/energy_completeness": 0.689851701259613, "metrics/hit_grouping/calorimeter/energy_purity": 0.6820441484451294, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3037954568862915, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8687909245491028, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6015152335166931, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21749776601791382, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6099028587341309, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6057920455932617, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27297699451446533, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8546350002288818, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6231138706207275, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.28165939450263977, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6272146701812744, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6413426995277405, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32884523272514343, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8109431862831116, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5704704523086548, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.36999115347862244, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.567919909954071, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5943395495414734, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4302869439125061, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8410588502883911, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6507967710494995, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31697291135787964, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6576128602027893, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6805639266967773, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36181876063346863, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8291333913803101, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7206035256385803, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.3961276710033417, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7147991061210632, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7327321171760559, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37153321504592896, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8652210235595703, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5287733674049377, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2038971483707428, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5325056910514832, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5227221250534058, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2790277302265167, "metrics/hit_grouping/calorimeter/purity": 0.668207585811615, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8101300597190857, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.16361868381500244, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1230769231915474, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.164333313703537, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12093385308980942, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8061054944992065, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2907911241054535, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10547320544719696, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28963592648506165, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28957799077033997, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24800455570220947, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8548134565353394, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6265055537223816, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2828567624092102, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6316092610359192, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6409532427787781, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3353554606437683, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7554872632026672, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38986849784851074, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5542263984680176, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5968420505523682, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5425881743431091, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8093528151512146, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2905215919017792, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.594200849533081, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6120395064353943, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40661805868148804, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6895105838775635, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4186028242111206, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4771365523338318, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5301898717880249, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6381034851074219, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7204568386077881, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3987341821193695, "metrics/hit_grouping/tracker/all/object_completeness": 0.5113615393638611, "metrics/hit_grouping/tracker/all/object_purity": 0.558150053024292, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5884186625480652, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004904577508568764, "metrics/hit_grouping/tracker/background_recall": 0.05046496540307999, "metrics/hit_grouping/tracker/completeness": 0.5228347182273865, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.41504666209220886, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7570176124572754, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37703871726989746, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5847619771957397, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6130974292755127, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5388610363006592, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6997165679931641, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4108128547668457, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.48182451725006104, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5454720854759216, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6274510025978088, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6613335013389587, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4340614974498749, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3798055946826935, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.44964510202407837, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6569579243659973, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7180353403091431, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4074406921863556, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5118487477302551, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.576662540435791, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5855499505996704, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6987476944923401, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4254327416419983, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48553815484046936, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5633795857429504, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6238349080085754, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7274287939071655, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3826972544193268, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5152673721313477, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5331136584281921, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5859283804893494, "metrics/hit_grouping/tracker/purity": 0.49835553765296936, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8943575024604797, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05734144151210785, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.17104661464691162, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16492150723934174, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07471763342618942, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7096999287605286, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28719544410705566, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4073502719402313, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.42791345715522766, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5142561197280884, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6980514526367188, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4578242003917694, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.569447934627533, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6264127492904663, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6666666865348816, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.6048735976219177, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7305197715759277, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8322635293006897, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6603521704673767, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.11914625018835068, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12954741716384888, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.4183292388916, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.31557697057724, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18513715267181396, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9266521334648132, "metrics/particle/class_1/pid_accuracy": 0.9770460724830627, "metrics/particle/class_2/efficiency": 0.5857079029083252, "metrics/particle/class_2/pid_accuracy": 0.8011875152587891, "metrics/particle/class_3/efficiency": 0.8235041499137878, "metrics/particle/class_3/pid_accuracy": 0.9809826612472534, "metrics/particle/class_4/efficiency": 0.7709335684776306, "metrics/particle/class_4/pid_accuracy": 0.6292135119438171, "metrics/particle/class_5/efficiency": 0.9580000042915344, "metrics/particle/class_5/pid_accuracy": 0.6931106448173523, "metrics/particle/count/bias_mean": -0.611328125, "metrics/particle/count/mae": 2.888671875, "metrics/particle/count/prediction_mean": 69.44466400146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05455341935157776, "metrics/particle/event/energy_response_mean": 0.9833349585533142, "metrics/particle/event/met_abs_error": 5.345176696777344, "metrics/particle/event/scalar_pt_relative_abs_error": 0.041364945471286774, "metrics/particle/event/scalar_pt_response_mean": 0.994122326374054, "metrics/particle/event/vector_pt_closure": 0.044275034219026566, "metrics/particle/matched/delta_eta_abs_mean": 0.012369937263429165, "metrics/particle/matched/delta_phi_abs_mean": 0.012888099066913128, "metrics/particle/matched/delta_r_mean": 0.020108185708522797, "metrics/particle/matched/energy_relative_abs_mean": 0.12075483053922653, "metrics/particle/matched/pid_accuracy": 0.9641626477241516, "metrics/particle/matched/pt_relative_abs_mean": 0.12199007719755173, "metrics/particle/matching/duplicate_fraction": 0.04120299592614174, "metrics/particle/matching/efficiency": 0.8523688316345215, "metrics/particle/matching/f1": 0.8561041355133057, "metrics/particle/matching/purity": 0.8598723411560059, "metrics/particle/matching/target_coverage_dr0p05": 0.816116213798523, "metrics/particle/matching/target_coverage_dr0p10": 0.8890582323074341, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6955745816230774, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8807501196861267, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019763296470046043, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11896248906850815, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11944487690925598, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017950937151908875, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15802153944969177, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0060836500488221645, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5730685591697693, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.028853444382548332, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6529799699783325, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9205.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19591836631298065, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7540761232376099, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8038052916526794, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6761912107467651, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09708187729120255, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.4769978523254395, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011899140663444996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8306094408035278, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03532867878675461, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.605398654937744, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10589.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8417154550552368, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8634180426597595, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.137012958526611, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1277506351470947, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.02128736861050129, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.23005267977714539, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.1642680168151855, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5225332379341125, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.690651893615723, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.020962590351700783, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1279383897781372, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12861546874046326, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028664475306868553, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16998082399368286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02568807266652584, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2936310768127441, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.03997969999909401, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6024186611175537, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2180.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.37042754888534546, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8228999376296997, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.642371416091919, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.5052751302719116, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1461339294910431, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.660852432250977, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05290418118238449, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1743149757385254, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12370628118515015, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.773281574249268, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2703.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7216153740882874, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.339974880218506, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.7205939292907715, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1051034927368164, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08618438243865967, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.01979769393801689, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.59752082824707, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5161411762237549, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.97700309753418, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01989506185054779, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11951787024736404, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12127263098955154, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025375664234161377, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16139189898967743, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00048064085422083735, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.196818545460701, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.016764206811785698, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6292276978492737, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18725.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.146178737282753, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8395448327064514, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6884506344795227, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5625198483467102, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0894089862704277, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.078701972961426, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027895981911569834, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.32052087783813477, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.024872245267033577, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3812376260757446, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21150.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8734897375106812, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.30681246519088745, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.110226631164551, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.045950889587402, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5215900540351868, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.34295058250427246, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.056140422821045, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2587156891822815, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.394652843475342, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01993206888437271, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11995773762464523, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12124550342559814, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023343944922089577, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1610102504491806, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004018598701804876, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39125335216522217, "metrics/particle/momentum/all/accepted/pz_bias": -0.004499154631048441, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4216587543487549, "metrics/particle/momentum/all/accepted/samples": 30110.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1790698915719986, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8119616508483887, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7987792491912842, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.671454668045044, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.0962197333574295, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.606246948242188, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009523256681859493, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6228300929069519, "metrics/particle/momentum/all/assigned/pz_bias": -0.014120277017354965, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.618959903717041, "metrics/particle/momentum/all/assigned/samples": 34442.0, "metrics/particle/momentum/all/efficiency": 0.8507091403007507, "metrics/particle/momentum/all/event/energy_bias": -3.896580457687378, "metrics/particle/momentum/all/event/p3_error_mean": 16.241127014160156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.524178981781006, "metrics/particle/momentum/all/event/px_bias": 0.41411828994750977, "metrics/particle/momentum/all/event/py_bias": 0.5928008556365967, "metrics/particle/momentum/all/event/pz_abs_mean": 12.319196701049805, "metrics/particle/momentum/all/event/pz_bias": -2.2973902225494385, "metrics/particle/momentum/all/event/pz_residual_rms": 21.03183937072754, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021240973845124245, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09357111901044846, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09569273144006729, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020235631614923477, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.13009248673915863, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003618272254243493, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3838731646537781, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005832515191286802, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.311830997467041, "metrics/particle/momentum/class_1/accepted/samples": 15477.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09216354042291641, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.22562989592552185, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2235579639673233, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12727178633213043, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04034651815891266, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.110889434814453, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005755134392529726, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.48388025164604187, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.006896943785250187, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7994894981384277, "metrics/particle/momentum/class_1/assigned/samples": 16507.0, "metrics/particle/momentum/class_1/efficiency": 0.9274884462356567, "metrics/particle/momentum/class_1/event/energy_bias": 3.1832613945007324, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.281635284423828, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.936063766479492, "metrics/particle/momentum/class_1/event/px_bias": -0.08462673425674438, "metrics/particle/momentum/class_1/event/py_bias": 0.7581157684326172, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.15121841430664, "metrics/particle/momentum/class_1/event/pz_bias": -1.1673272848129272, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.868473052978516, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02670261636376381, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17429089546203613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17675404250621796, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.003194737248122692, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21588754653930664, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0242774561047554, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.420000672340393, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05804620683193207, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.661158800125122, "metrics/particle/momentum/class_2/accepted/samples": 1730.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5712145566940308, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6306397914886475, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7891418933868408, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.46574667096138, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2381511926651001, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.396074295043945, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04817861318588257, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.29133939743042, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06837675720453262, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.830009937286377, "metrics/particle/momentum/class_2/assigned/samples": 2553.0, "metrics/particle/momentum/class_2/efficiency": 0.5811219215393066, "metrics/particle/momentum/class_2/event/energy_bias": -8.83832836151123, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.494237899780273, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.239597797393799, "metrics/particle/momentum/class_2/event/px_bias": 0.8145139217376709, "metrics/particle/momentum/class_2/event/py_bias": -0.20205694437026978, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.484903335571289, "metrics/particle/momentum/class_2/event/pz_bias": -0.938951849937439, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.86945915222168, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017349714413285255, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1446821242570877, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14491474628448486, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03018266335129738, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18463343381881714, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001363599905744195, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23689059913158417, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0073380540125072, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8317232131958008, "metrics/particle/momentum/class_3/accepted/samples": 12467.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20784030854701996, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4598419666290283, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4132884740829468, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2844066619873047, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13304634392261505, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.003185272216797, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006793569773435593, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4734130799770355, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.032137431204319, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.478755474090576, "metrics/particle/momentum/class_3/assigned/samples": 14867.0, "metrics/particle/momentum/class_3/efficiency": 0.8198198080062866, "metrics/particle/momentum/class_3/event/energy_bias": 2.4011194705963135, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.587395668029785, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.270356178283691, "metrics/particle/momentum/class_3/event/px_bias": -0.10296103358268738, "metrics/particle/momentum/class_3/event/py_bias": -0.10692405700683594, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.406077861785889, "metrics/particle/momentum/class_3/event/pz_bias": -0.3760724663734436, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.876408576965332, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02372686378657818, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.13797982037067413, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13277964293956757, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05163109675049782, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17743031680583954, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007042253389954567, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6833997368812561, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04166325554251671, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.835250735282898, "metrics/particle/momentum/class_4/accepted/samples": 284.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2546814978122711, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.5382916927337646, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1654093265533447, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0631625652313232, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.16011236608028412, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.84136390686035, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01123595517128706, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.891262948513031, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.12224507331848145, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.853419542312622, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.7802197933197021, "metrics/particle/momentum/class_4/event/energy_bias": -0.6077929139137268, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.346856117248535, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5501880645751953, "metrics/particle/momentum/class_4/event/px_bias": 0.043620456010103226, "metrics/particle/momentum/class_4/event/py_bias": -0.04589211940765381, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.497197151184082, "metrics/particle/momentum/class_4/event/pz_bias": 0.3315219283103943, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.741756916046143, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014310899190604687, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1267470419406891, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12841932475566864, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.028129443526268005, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16386599838733673, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5488982200622559, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3004492521286011, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0902228355407715, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04555011913180351, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15085116028785706, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.15320147573947906, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.041227325797080994, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03144654259085655, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2345828413963318, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03144654259085655, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6276097297668457, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.16190002858638763, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.331634998321533, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.955974817276001, "metrics/particle/momentum/class_5/event/energy_bias": -0.03484024107456207, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7348003387451172, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3526461124420166, "metrics/particle/momentum/class_5/event/px_bias": -0.2564283311367035, "metrics/particle/momentum/class_5/event/py_bias": 0.18955831229686737, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8659041523933411, "metrics/particle/momentum/class_5/event/pz_bias": -0.14656074345111847, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.888350248336792, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2952289283275604, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2970345616340637, "metrics/set_slot/angular_recall_all_dr0p10": 0.9633849263191223, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8732412457466125, "metrics/set_slot/soft_count_bias_mean": 0.47888365387916565}, "step_train_time": 5.12461519241333, "step_valid_time": 36486.515681028366, "step_total_time": 36491.6402964592, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001919574075624, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10591008339127361, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2897, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616782292699042, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9597757995384109, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607260726072607, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040224200461589185, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9570531879748926, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9551599076821629, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9561056105610561, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616782292699042, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988689314166375, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10403329091239799, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2256, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2155, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9526967285587975, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552304964539007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9539619300575476, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04476950354609929, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496021220159151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9521276595744681, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.950863213811421, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9526967285587975, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9975721645859963, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1044257566585104, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9482352941176471, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434256730394069, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05176470588235294, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9355590062111802, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9450980392156862, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9403043308622708, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386645962732919} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_42000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..0d0ce6a796c1d69fcf1f31d551ee9854fba01442 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9897669130187607, "train": {"Classification_binary": 0.15309913456439972, "Cardinality": 0.11317645013332367, "Classification": 0.061822377145290375, "Regression_pt": 0.08294432610273361, "Regression_eta": 0.001206306042149663, "Regression_sin_phi": 0.0006055724807083607, "Regression_cos_phi": 0.000591576041188091, "Regression_energy": 0.08207132667303085, "HitGrouping": 0.14181675016880035, "Auxiliary": 0.18211548030376434, "Total": 0.8377552032470703, "Total_unweighted": 0.3823406190495007}, "train_group": {}, "valid": {"Classification_binary": 0.12862197558085123, "Cardinality": 0.09611111879348755, "Classification": 0.06355387965838115, "Regression_pt": 0.08850433429082234, "Regression_eta": 0.001141513076921304, "Regression_sin_phi": 0.0006687034231921037, "Regression_cos_phi": 0.0006670924679686626, "Regression_energy": 0.08809884389241536, "HitGrouping": 0.14162186781565347, "Auxiliary": 0.17011916637420654, "Total": 0.7996991475423177, "Total_unweighted": 0.37125634239055216, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.850450873374939, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6143812537193298, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28034383058547974, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.620664119720459, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6261031627655029, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3277604579925537, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7985002398490906, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5643588304519653, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3043036162853241, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5725545883178711, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5877364277839661, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4191799461841583, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8589878678321838, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5876713395118713, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2593740224838257, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5920222401618958, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5971694588661194, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3106084167957306, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8508152961730957, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5941753387451172, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2701519727706909, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5994927883148193, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6056340336799622, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3258662521839142, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02686595357954502, "metrics/hit_grouping/calorimeter/background_recall": 0.6528583765029907, "metrics/hit_grouping/calorimeter/completeness": 0.6747420430183411, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6859766840934753, "metrics/hit_grouping/calorimeter/energy_purity": 0.678239107131958, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30741602182388306, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8661972284317017, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5964317321777344, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21576589345932007, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6050605177879333, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6035862565040588, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27685877680778503, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8559430837631226, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6176877021789551, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2769286632537842, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6223389506340027, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6293241381645203, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3265404999256134, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8129128813743591, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.564897358417511, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37820756435394287, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5639000535011292, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5852811336517334, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4288964867591858, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8415402173995972, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6464197039604187, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3182809054851532, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6542825102806091, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6761907339096069, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3657427728176117, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8354592323303223, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7178085446357727, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.41705912351608276, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7148564457893372, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7143312096595764, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.35269492864608765, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8627514839172363, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5221889019012451, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1996116191148758, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5258281230926514, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5158717632293701, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27956342697143555, "metrics/hit_grouping/calorimeter/purity": 0.6655397415161133, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7903797030448914, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1387268602848053, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10923077166080475, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.13765811920166016, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.10187242925167084, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04461538419127464, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7974913120269775, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.279424250125885, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10376282781362534, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2787962853908539, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2716456651687622, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2428734302520752, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8551703095436096, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6219860911369324, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28315189480781555, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6277815699577332, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6355808973312378, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3366343080997467, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7551419138908386, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39554691314697266, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5557447075843811, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.598510205745697, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5430364608764648, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8118616342544556, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.29332584142684937, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.593906581401825, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.602281928062439, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.39315760135650635, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6892346739768982, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.417815238237381, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.47408154606819153, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5271733403205872, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6375522017478943, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7203935980796814, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40029284358024597, "metrics/hit_grouping/tracker/all/object_completeness": 0.50998455286026, "metrics/hit_grouping/tracker/all/object_purity": 0.5560466647148132, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5870961546897888, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006075819488614798, "metrics/hit_grouping/tracker/background_recall": 0.057928092777729034, "metrics/hit_grouping/tracker/completeness": 0.5223459005355835, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.42165860533714294, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7562000751495361, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37840548157691956, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5808948278427124, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6114259958267212, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5383143424987793, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6989598274230957, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4110032320022583, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.48028090596199036, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5406745672225952, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6251665949821472, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6636813879013062, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4375, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.384133517742157, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.44944506883621216, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6549353003501892, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7213922739028931, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40600287914276123, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5130599141120911, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5758410096168518, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.579798698425293, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7056422233581543, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4427430033683777, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48779594898223877, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5513399243354797, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.606524646282196, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7216863036155701, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3853898346424103, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5098797082901001, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5310874581336975, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.593186616897583, "metrics/hit_grouping/tracker/purity": 0.4984763562679291, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8898822069168091, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.059947870671749115, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16504473984241486, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16247984766960144, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07819287478923798, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7058678865432739, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2742353677749634, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4022689759731293, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4185695946216583, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5184032917022705, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6990169286727905, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4608936905860901, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5691211819648743, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6251797676086426, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.664069414138794, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5857163667678833, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7202631831169128, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.81654953956604, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6560622453689575, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09998905658721924, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1252574324607849, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.81474494934082, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3184436559677124, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18800383806228638, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9311436414718628, "metrics/particle/class_1/pid_accuracy": 0.9791967868804932, "metrics/particle/class_2/efficiency": 0.5848104357719421, "metrics/particle/class_2/pid_accuracy": 0.7995396256446838, "metrics/particle/class_3/efficiency": 0.8193637132644653, "metrics/particle/class_3/pid_accuracy": 0.9806247353553772, "metrics/particle/class_4/efficiency": 0.7911453247070312, "metrics/particle/class_4/pid_accuracy": 0.5997567176818848, "metrics/particle/class_5/efficiency": 0.9700000286102295, "metrics/particle/class_5/pid_accuracy": 0.6762886643409729, "metrics/particle/count/bias_mean": -1.2493489980697632, "metrics/particle/count/mae": 3.080078125, "metrics/particle/count/prediction_mean": 68.806640625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05429146811366081, "metrics/particle/event/energy_response_mean": 0.9848925471305847, "metrics/particle/event/met_abs_error": 4.979262351989746, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04250757023692131, "metrics/particle/event/scalar_pt_response_mean": 0.9959316849708557, "metrics/particle/event/vector_pt_closure": 0.0430334210395813, "metrics/particle/matched/delta_eta_abs_mean": 0.012759610079228878, "metrics/particle/matched/delta_phi_abs_mean": 0.012944431975483894, "metrics/particle/matched/delta_r_mean": 0.020402804017066956, "metrics/particle/matched/energy_relative_abs_mean": 0.12018810957670212, "metrics/particle/matched/pid_accuracy": 0.9645865559577942, "metrics/particle/matched/pt_relative_abs_mean": 0.11959676444530487, "metrics/particle/matching/duplicate_fraction": 0.03869917616248131, "metrics/particle/matching/efficiency": 0.8528613448143005, "metrics/particle/matching/f1": 0.8605345487594604, "metrics/particle/matching/purity": 0.8683471083641052, "metrics/particle/matching/target_coverage_dr0p05": 0.8147779703140259, "metrics/particle/matching/target_coverage_dr0p10": 0.8858985304832458, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7021727561950684, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8804248571395874, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02032708190381527, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11809786409139633, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1167459636926651, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009122896939516068, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15601776540279388, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005626487545669079, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5649343132972717, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05057796090841293, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5699604749679565, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9242.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18659912049770355, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9105310440063477, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.827465295791626, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6901755928993225, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09263297915458679, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.031953811645508, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011188015341758728, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8083315491676331, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.023998716846108437, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4718244075775146, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10547.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8450987339019775, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.40447932481765747, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.727830410003662, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.962156295776367, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22572758793830872, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5003330111503601, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.821334362030029, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0116627216339111, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.393712043762207, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02075830101966858, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12783518433570862, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1262437403202057, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01108736451715231, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16511458158493042, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022079117596149445, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2315659523010254, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1226034015417099, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4036223888397217, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2174.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3317202627658844, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8408520817756653, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.3779016733169556, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.2301982641220093, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1293136179447174, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.270349502563477, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.049298446625471115, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 1.9995521306991577, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.059199877083301544, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.375603675842285, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2637.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7196292877197266, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.4789295196533203, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.3824543952941895, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0592933893203735, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07510171085596085, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10300247371196747, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.242752552032471, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3574860095977783, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.316852569580078, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020277753472328186, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11933878809213638, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11917774379253387, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019899681210517883, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1596672236919403, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003733333433046937, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19498345255851746, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.033787257969379425, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6364107728004456, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18750.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1467238962650299, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7341196537017822, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.558234691619873, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42978352308273315, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08542832732200623, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.262257099151611, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002846029819920659, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3122550845146179, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01659264601767063, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.312747597694397, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21082.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8746559619903564, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.1753917634487152, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.604927062988281, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.438721179962158, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.05305173993110657, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.03907284140586853, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2088847160339355, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4780031442642212, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.688767910003662, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020327499136328697, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11957092583179474, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11894195526838303, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01596289686858654, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15895962715148926, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0035470398142933846, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.38303014636039734, "metrics/particle/momentum/all/accepted/pz_bias": -0.04533224180340767, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3571151494979858, "metrics/particle/momentum/all/accepted/samples": 30166.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17323410511016846, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7966324687004089, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7041821479797363, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.571528971195221, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09102316945791245, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.595464706420898, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008988501504063606, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5947952270507812, "metrics/particle/momentum/all/assigned/pz_bias": -0.02215113304555416, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.2725982666015625, "metrics/particle/momentum/all/assigned/samples": 34266.0, "metrics/particle/momentum/all/efficiency": 0.8522913455963135, "metrics/particle/momentum/all/event/energy_bias": -4.058800220489502, "metrics/particle/momentum/all/event/p3_error_mean": 14.923033714294434, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.8454976081848145, "metrics/particle/momentum/all/event/px_bias": 0.20367759466171265, "metrics/particle/momentum/all/event/py_bias": 0.43640339374542236, "metrics/particle/momentum/all/event/pz_abs_mean": 11.542672157287598, "metrics/particle/momentum/all/event/pz_bias": -2.8471519947052, "metrics/particle/momentum/all/event/pz_residual_rms": 18.8129825592041, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021473299711942673, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09178828448057175, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09196504205465317, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014139959588646889, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12585319578647614, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0028966849204152822, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3733506202697754, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0568072535097599, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2340383529663086, "metrics/particle/momentum/class_1/accepted/samples": 15535.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08322342485189438, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19384066760540009, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.20424067974090576, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10582166165113449, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03720563277602196, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7619863748550415, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005037630442529917, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4664493203163147, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04268999397754669, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6722278594970703, "metrics/particle/momentum/class_1/assigned/samples": 16476.0, "metrics/particle/momentum/class_1/efficiency": 0.9309642314910889, "metrics/particle/momentum/class_1/event/energy_bias": 3.0686206817626953, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.62811279296875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.5875139236450195, "metrics/particle/momentum/class_1/event/px_bias": -0.08820939809083939, "metrics/particle/momentum/class_1/event/py_bias": 0.3201856017112732, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.74454402923584, "metrics/particle/momentum/class_1/event/pz_bias": -2.1719260215759277, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.340702056884766, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02741808071732521, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1744726002216339, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1753660887479782, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.002639556070789695, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21637701988220215, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022247575223445892, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.331857442855835, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11126110702753067, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.354708433151245, "metrics/particle/momentum/class_2/accepted/samples": 1753.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.50336754322052, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.559532642364502, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6551158428192139, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3560008406639099, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2165479063987732, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9687018394470215, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04790182039141655, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.088390827178955, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.01888761669397354, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.2144880294799805, "metrics/particle/momentum/class_2/assigned/samples": 2526.0, "metrics/particle/momentum/class_2/efficiency": 0.588847815990448, "metrics/particle/momentum/class_2/event/energy_bias": -8.354135513305664, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.272163391113281, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.066127777099609, "metrics/particle/momentum/class_2/event/px_bias": 0.33285975456237793, "metrics/particle/momentum/class_2/event/py_bias": -0.1579970121383667, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.244075775146484, "metrics/particle/momentum/class_2/event/pz_bias": -0.47296929359436035, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.782848358154297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017856981605291367, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14596262574195862, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14417743682861328, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019329406321048737, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18397463858127594, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016086222603917122, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2417290061712265, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02317236363887787, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9090986251831055, "metrics/particle/momentum/class_3/accepted/samples": 12433.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21696510910987854, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.503782033920288, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2694613933563232, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1263233423233032, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12914378941059113, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.431432723999023, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006575825158506632, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4636894464492798, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0043869283981621265, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.946656346321106, "metrics/particle/momentum/class_3/assigned/samples": 14751.0, "metrics/particle/momentum/class_3/efficiency": 0.8175839781761169, "metrics/particle/momentum/class_3/event/energy_bias": 1.8871389627456665, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.777877330780029, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 3.937872886657715, "metrics/particle/momentum/class_3/event/px_bias": 0.06921502947807312, "metrics/particle/momentum/class_3/event/py_bias": 0.24991998076438904, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.73085355758667, "metrics/particle/momentum/class_3/event/pz_bias": -0.49121057987213135, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.283960342407227, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02409491129219532, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14045540988445282, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13666187226772308, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.048249512910842896, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17903032898902893, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006896551698446274, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6564542651176453, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2039211392402649, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6756808757781982, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.24508658051490784, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3696926832199097, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0208576917648315, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.9088131189346313, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14406779408454895, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.15748405456543, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9662445187568665, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06990324705839157, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.773817300796509, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7967032790184021, "metrics/particle/momentum/class_4/event/energy_bias": -0.4336104393005371, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5658278465270996, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6833043098449707, "metrics/particle/momentum/class_4/event/px_bias": 0.07744795083999634, "metrics/particle/momentum/class_4/event/py_bias": -0.06329029053449631, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.654220461845398, "metrics/particle/momentum/class_4/event/pz_bias": 0.3464539647102356, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.278967380523682, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016415206715464592, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12715761363506317, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12722033262252808, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.018906304612755775, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16055713593959808, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.006451612804085016, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.444841742515564, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.3934592604637146, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.643357753753662, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03856322169303894, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1455966830253601, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.14092546701431274, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.032195087522268295, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.01886792480945587, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.19950710237026215, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.012578615918755531, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.502047061920166, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3014214336872101, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.8338701725006104, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9748427867889404, "metrics/particle/momentum/class_5/event/energy_bias": -0.2268143594264984, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5927438735961914, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.229063630104065, "metrics/particle/momentum/class_5/event/px_bias": -0.18763579428195953, "metrics/particle/momentum/class_5/event/py_bias": 0.08758511394262314, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.806902289390564, "metrics/particle/momentum/class_5/event/pz_bias": -0.05749993398785591, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.603933334350586, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.29683032631874084, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2980216145515442, "metrics/set_slot/angular_recall_all_dr0p10": 0.9645651578903198, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8705276846885681, "metrics/set_slot/soft_count_bias_mean": -0.31885766983032227}, "step_train_time": 5.061755895614624, "step_valid_time": 39572.216128349304, "step_total_time": 39577.27788448334, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002679352613058, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1009962657518122, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2913, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2895, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9623389494549058, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636123056566325, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9629752066115702, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03638769434336751, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956392467789891, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9576579556731724, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9570247933884297, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9623389494549058, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988046749290727, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10480145411915831, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2258, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2154, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522546419098143, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953941541186891, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9530973451327434, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046058458813108945, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496021220159151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9512843224092117, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9504424778761061, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522546419098143, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9934490358577812, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10125961357618407, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1214, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9425465838509317, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9469578783151326, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9447470817120622, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0530421216848674, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9430577223088924, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9408560311284047, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9425465838509317} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_43000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..4fb4baa2436868c656fcad7ff1697c6718e28e6c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0371423157096835, "train": {"Classification_binary": 0.12480772286653519, "Cardinality": 0.08659327775239944, "Classification": 0.06536804139614105, "Regression_pt": 0.08119822293519974, "Regression_eta": 0.0010432826820760965, "Regression_sin_phi": 0.0006665427936241031, "Regression_cos_phi": 0.0005840484518557787, "Regression_energy": 0.08089809119701385, "HitGrouping": 0.14089158177375793, "Auxiliary": 0.16042307019233704, "Total": 0.7600861191749573, "Total_unweighted": 0.3545659523224458}, "train_group": {}, "valid": {"Classification_binary": 0.12861094872156778, "Cardinality": 0.09310012062390645, "Classification": 0.0641939143339793, "Regression_pt": 0.08671859900156657, "Regression_eta": 0.001074241551881035, "Regression_sin_phi": 0.0006630336865782738, "Regression_cos_phi": 0.0006486293083677689, "Regression_energy": 0.0862839122613271, "HitGrouping": 0.1410375734170278, "Auxiliary": 0.16695435841878256, "Total": 0.787672201792399, "Total_unweighted": 0.3681932788652678, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8479514122009277, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6169322729110718, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28062719106674194, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6232033371925354, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6324112415313721, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3339945077896118, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7969987392425537, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5616664886474609, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3032870292663574, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.570087194442749, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5851021409034729, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4107082486152649, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8617597818374634, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.6044520139694214, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2620597183704376, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.60820472240448, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6170770525932312, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3103501796722412, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8515075445175171, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.6046304106712341, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27173250913619995, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6096121668815613, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6191434264183044, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3269604742527008, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026668112725019455, "metrics/hit_grouping/calorimeter/background_recall": 0.6498602628707886, "metrics/hit_grouping/calorimeter/completeness": 0.6761686205863953, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6878128051757812, "metrics/hit_grouping/calorimeter/energy_purity": 0.6800620555877686, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3052545189857483, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.86798495054245, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6099097728729248, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21869215369224548, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6181221008300781, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6176844835281372, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2776351273059845, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.854246199131012, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6253874897956848, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.28008249402046204, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6304603815078735, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6430680751800537, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33017951250076294, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.813776433467865, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5718246102333069, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37530022859573364, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5698738098144531, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5972998738288879, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.42801162600517273, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8410691618919373, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6494429111480713, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31809404492378235, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6569063663482666, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6802984476089478, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36536905169487, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8357278108596802, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7146192789077759, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4107796847820282, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7119048833847046, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7180265784263611, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.35269492864608765, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8647487163543701, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.54237961769104, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20409803092479706, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5456781983375549, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5407444834709167, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2823757827281952, "metrics/hit_grouping/calorimeter/purity": 0.6668112277984619, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8083174824714661, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1605684906244278, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10923077166080475, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15753114223480225, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.11518402397632599, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8155463933944702, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3170192837715149, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11060433089733124, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3165217638015747, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.31228572130203247, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24458381533622742, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.854496419429779, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6306374073028564, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2844635248184204, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6361052393913269, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6475341320037842, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33784759044647217, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7505815029144287, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3930065631866455, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5552794337272644, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6018909811973572, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5572324991226196, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8114394545555115, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28883904218673706, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.597589910030365, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6169207692146301, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40044867992401123, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6894577145576477, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42151689529418945, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4793400466442108, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5304431915283203, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6382610201835632, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7190503478050232, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40133196115493774, "metrics/hit_grouping/tracker/all/object_completeness": 0.5133011937141418, "metrics/hit_grouping/tracker/all/object_purity": 0.5603089332580566, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5926223397254944, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005291941110044718, "metrics/hit_grouping/tracker/background_recall": 0.04714801907539368, "metrics/hit_grouping/tracker/completeness": 0.5257256031036377, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4191560447216034, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7554153800010681, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3805922567844391, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5839911103248596, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6153340339660645, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5439635515213013, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6957221627235413, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4136683940887451, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.48871469497680664, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5514870285987854, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6356368064880371, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6631109714508057, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43426376581192017, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3825026750564575, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4475341737270355, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6549353003501892, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7178210616111755, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40501436591148376, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5134822130203247, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5782895684242249, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5870776176452637, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7037706971168518, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4420772194862366, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4992373585700989, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5519656538963318, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6111850738525391, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7233384251594543, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3893701732158661, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5155383944511414, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5383519530296326, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5965815782546997, "metrics/hit_grouping/tracker/purity": 0.5013062357902527, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8964018821716309, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.06168549135327339, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.17026308178901672, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16171963512897491, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07384882867336273, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7069480419158936, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2882322371006012, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40583354234695435, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4262787103652954, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5189217329025269, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6963292360305786, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46036243438720703, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.572151243686676, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6297320127487183, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6715070009231567, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5983388423919678, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7205743193626404, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.819301962852478, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6576424241065979, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.11261151731014252, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1268376111984253, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.612030029296875, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3342728316783905, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2038329839706421, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9319746494293213, "metrics/particle/class_1/pid_accuracy": 0.9772621393203735, "metrics/particle/class_2/efficiency": 0.5701144337654114, "metrics/particle/class_2/pid_accuracy": 0.8101141452789307, "metrics/particle/class_3/efficiency": 0.8277090191841125, "metrics/particle/class_3/pid_accuracy": 0.981986403465271, "metrics/particle/class_4/efficiency": 0.7921078205108643, "metrics/particle/class_4/pid_accuracy": 0.6075333952903748, "metrics/particle/class_5/efficiency": 0.9639999866485596, "metrics/particle/class_5/pid_accuracy": 0.6763485670089722, "metrics/particle/count/bias_mean": -0.7005208134651184, "metrics/particle/count/mae": 2.8645832538604736, "metrics/particle/count/prediction_mean": 69.35546875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.055743876844644547, "metrics/particle/event/energy_response_mean": 0.9780404567718506, "metrics/particle/event/met_abs_error": 5.274002552032471, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043716493993997574, "metrics/particle/event/scalar_pt_response_mean": 0.9865778088569641, "metrics/particle/event/vector_pt_closure": 0.04376190900802612, "metrics/particle/matched/delta_eta_abs_mean": 0.011919105425477028, "metrics/particle/matched/delta_phi_abs_mean": 0.012584853917360306, "metrics/particle/matched/delta_r_mean": 0.01950196921825409, "metrics/particle/matched/energy_relative_abs_mean": 0.11928383260965347, "metrics/particle/matched/pid_accuracy": 0.9651355743408203, "metrics/particle/matched/pt_relative_abs_mean": 0.11936867982149124, "metrics/particle/matching/duplicate_fraction": 0.041744109243154526, "metrics/particle/matching/efficiency": 0.8556307554244995, "metrics/particle/matching/f1": 0.859930157661438, "metrics/particle/matching/purity": 0.864272952079773, "metrics/particle/matching/target_coverage_dr0p05": 0.8214318752288818, "metrics/particle/matching/target_coverage_dr0p10": 0.890545129776001, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.707144558429718, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8836310505867004, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019236775115132332, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11797785758972168, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11789267510175705, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004857733845710754, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1568000465631485, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005612520035356283, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.563636064529419, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.042765844613313675, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5894269943237305, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9265.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1837211400270462, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8361809253692627, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7535354495048523, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.612207293510437, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09577783942222595, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.413203716278076, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012184754945337772, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8386783003807068, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.00844191387295723, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7591450214385986, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10587.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8472018837928772, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.7035824060440063, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.136654376983643, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1147100925445557, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.04124855250120163, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.18759030103683472, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.170923233032227, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.8560558557510376, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.080705642700195, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021420640870928764, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12381330132484436, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12445686012506485, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012908508069813251, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1636923849582672, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022653721272945404, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2819479703903198, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0095860306173563, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7390782833099365, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2163.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3729386031627655, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9265560507774353, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2247812747955322, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.072692394256592, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.14104478061199188, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 26.21395492553711, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05223880708217621, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1709706783294678, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.11452734470367432, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.599743366241455, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2680.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7159880995750427, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.9498605728149414, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.4091620445251465, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1245124340057373, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.063809335231781, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08845415711402893, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.2657470703125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4450174570083618, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.247699737548828, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019209248945116997, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11784672737121582, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1181127056479454, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008064432069659233, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15745744109153748, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00047857066965661943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19331470131874084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013702629134058952, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6135368347167969, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18806.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14042282104492188, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7563953995704651, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6205617189407349, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48320820927619934, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08689276874065399, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.310206413269043, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002601579762995243, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3109876811504364, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.014146418310701847, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3383842706680298, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21141.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8772682547569275, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.1923136711120605, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.692788600921631, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.61871862411499, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.11841188371181488, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.16908156871795654, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0306549072265625, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.07403277605772018, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.413979530334473, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019375892356038094, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11831377446651459, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11849914491176605, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007428316865116358, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15771108865737915, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0036382880061864853, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3846800923347473, "metrics/particle/momentum/all/accepted/pz_bias": -0.022314343601465225, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.417219638824463, "metrics/particle/momentum/all/accepted/samples": 30234.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17185565829277039, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7941983342170715, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7864271998405457, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6467030048370361, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.0938444584608078, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.950484275817871, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009416414424777031, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6182248592376709, "metrics/particle/momentum/all/assigned/pz_bias": -0.02020975761115551, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6144938468933105, "metrics/particle/momentum/all/assigned/samples": 34408.0, "metrics/particle/momentum/all/efficiency": 0.8542125821113586, "metrics/particle/momentum/all/event/energy_bias": -5.845757484436035, "metrics/particle/momentum/all/event/p3_error_mean": 15.108309745788574, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.122193336486816, "metrics/particle/momentum/all/event/px_bias": -0.09585113823413849, "metrics/particle/momentum/all/event/py_bias": -0.06994542479515076, "metrics/particle/momentum/all/event/pz_abs_mean": 11.419560432434082, "metrics/particle/momentum/all/event/pz_bias": -2.2270402908325195, "metrics/particle/momentum/all/event/pz_residual_rms": 19.86310386657715, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.020549967885017395, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09036974608898163, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09116116911172867, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0031480747275054455, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1240667849779129, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0028264918364584446, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37508031725883484, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.031519316136837006, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2416760921478271, "metrics/particle/momentum/class_1/accepted/samples": 15567.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08510163426399231, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20602338016033173, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22789771854877472, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1126253604888916, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03647821769118309, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6452198028564453, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005029388703405857, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4710538387298584, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03176378458738327, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6759157180786133, "metrics/particle/momentum/class_1/assigned/samples": 16503.0, "metrics/particle/momentum/class_1/efficiency": 0.9328818917274475, "metrics/particle/momentum/class_1/event/energy_bias": 0.04649687185883522, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.32715129852295, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.424466609954834, "metrics/particle/momentum/class_1/event/px_bias": -0.4188404083251953, "metrics/particle/momentum/class_1/event/py_bias": 0.190082848072052, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.439020156860352, "metrics/particle/momentum/class_1/event/pz_bias": -1.4522873163223267, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.280305862426758, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026339571923017502, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1733522117137909, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17545004189014435, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.014312070794403553, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21547086536884308, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02369668334722519, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4148236513137817, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0701497420668602, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6247498989105225, "metrics/particle/momentum/class_2/accepted/samples": 1688.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.537112295627594, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6804750561714172, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8542452454566956, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5483007431030273, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2409019023180008, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.222261428833008, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05221518874168396, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.314141273498535, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.008992880582809448, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.862901210784912, "metrics/particle/momentum/class_2/assigned/samples": 2528.0, "metrics/particle/momentum/class_2/efficiency": 0.5670138001441956, "metrics/particle/momentum/class_2/event/energy_bias": -7.895150661468506, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.771929740905762, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.209715366363525, "metrics/particle/momentum/class_2/event/px_bias": 0.7232141494750977, "metrics/particle/momentum/class_2/event/py_bias": -0.3200035095214844, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.642786026000977, "metrics/particle/momentum/class_2/event/pz_bias": -0.06744301319122314, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.165380477905273, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016946684569120407, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14496822655200958, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14421238005161285, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019027331843972206, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18308515846729279, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013567438581958413, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23595838248729706, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0020168607588857412, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9792603254318237, "metrics/particle/momentum/class_3/accepted/samples": 12530.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2049878090620041, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.449275016784668, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3822039365768433, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2431893348693848, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13232819736003876, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.676655769348145, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006596217397600412, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.47354891896247864, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.006478844676166773, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.539670944213867, "metrics/particle/momentum/class_3/assigned/samples": 14857.0, "metrics/particle/momentum/class_3/efficiency": 0.8239626288414001, "metrics/particle/momentum/class_3/event/energy_bias": 3.13677978515625, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.642943382263184, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.247314929962158, "metrics/particle/momentum/class_3/event/px_bias": -0.1209227591753006, "metrics/particle/momentum/class_3/event/py_bias": -0.22634388506412506, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.510992527008057, "metrics/particle/momentum/class_3/event/pz_bias": -0.3648694157600403, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.308154106140137, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024306632578372955, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1404430866241455, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.135924831032753, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.041870903223752975, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1819053441286087, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010135134682059288, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6500499844551086, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.23268374800682068, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.693261742591858, "metrics/particle/momentum/class_4/accepted/samples": 296.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26591938734054565, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7679088115692139, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5764826536178589, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4585069417953491, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1325966864824295, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.806583404541016, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013812154531478882, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9654762148857117, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.019157789647579193, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.494486093521118, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.8131868243217468, "metrics/particle/momentum/class_4/event/energy_bias": -0.7883131504058838, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2821474075317383, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5696816444396973, "metrics/particle/momentum/class_4/event/px_bias": 0.07307260483503342, "metrics/particle/momentum/class_4/event/py_bias": 0.12530548870563507, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.369873285293579, "metrics/particle/momentum/class_4/event/pz_bias": 0.0566687136888504, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.5497565269470215, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01249343529343605, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12856966257095337, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13217535614967346, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.008962360210716724, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16598978638648987, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03921568766236305, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6623938083648682, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.7136013507843018, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.315757989883423, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05816911160945892, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.21941952407360077, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2073947638273239, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.056673694401979446, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6665501594543457, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.037974681705236435, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6639858484268188, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6619778275489807, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.283142328262329, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": -0.3455701768398285, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.692286491394043, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.292426347732544, "metrics/particle/momentum/class_5/event/px_bias": -0.3523748517036438, "metrics/particle/momentum/class_5/event/py_bias": 0.16101369261741638, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.884027361869812, "metrics/particle/momentum/class_5/event/pz_bias": -0.3991093933582306, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.963315010070801, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.28801390528678894, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.28899261355400085, "metrics/set_slot/angular_recall_all_dr0p10": 0.9655502438545227, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8741891980171204, "metrics/set_slot/soft_count_bias_mean": 0.36334148049354553}, "step_train_time": 5.368834972381592, "step_valid_time": 42704.813645362854, "step_total_time": 42710.182480573654, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9901451421399291, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10154953246376641, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2914, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2902, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9626693095474067, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636243386243386, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9631465873409354, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036375661375661374, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9587049884373967, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9596560846560847, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9591803007767311, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9626693095474067, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9902777533614431, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10284632308629638, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2256, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2163, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2158, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562334217506632, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9587765957446809, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9575033200531209, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041223404255319146, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9540229885057471, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9565602836879432, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9552899513058876, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562334217506632, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9887463823210838, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10052159664293248, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1198, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1196, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9301242236024845, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9366692728694292, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9333852746396571, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06333072713057075, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9285714285714286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9351055512118843, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9318270354499416, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9301242236024845} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_44000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..078f8d2cc9211467d4a963b83710ef87fe9601b4 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0845177184006065, "train": {"Classification_binary": 0.13297049701213837, "Cardinality": 0.10137820988893509, "Classification": 0.05880621075630188, "Regression_pt": 0.0842258632183075, "Regression_eta": 0.0012399217812344432, "Regression_sin_phi": 0.0006547508528456092, "Regression_cos_phi": 0.000623535830527544, "Regression_energy": 0.0843755230307579, "HitGrouping": 0.14142097532749176, "Auxiliary": 0.16585971415042877, "Total": 0.7962790131568909, "Total_unweighted": 0.36289630248211324}, "train_group": {}, "valid": {"Classification_binary": 0.12576373418172201, "Cardinality": 0.09336075186729431, "Classification": 0.06348487238089244, "Regression_pt": 0.08584603667259216, "Regression_eta": 0.0011381404474377632, "Regression_sin_phi": 0.0006499881080041329, "Regression_cos_phi": 0.0006320031049350897, "Regression_energy": 0.08541767795880635, "HitGrouping": 0.13922378420829773, "Auxiliary": 0.16631789008776346, "Total": 0.7808929284413656, "Total_unweighted": 0.36293245285438996, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8496670126914978, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6249762773513794, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2780768871307373, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6306535601615906, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6423670649528503, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.32898837327957153, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7992383241653442, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5670772790908813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.28973230719566345, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5748125910758972, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.597990095615387, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41240257024765015, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8573698401451111, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.6023173332214355, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2633509039878845, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.6062775254249573, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.616723358631134, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31732258200645447, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8496769666671753, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.6064479351043701, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.270455926656723, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6112992763519287, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6232950091362, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3296048641204834, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026726549491286278, "metrics/hit_grouping/calorimeter/background_recall": 0.6510323882102966, "metrics/hit_grouping/calorimeter/completeness": 0.6821596026420593, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6936284899711609, "metrics/hit_grouping/calorimeter/energy_purity": 0.6857461929321289, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30058303475379944, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8676573634147644, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6082689166069031, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21516871452331543, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.616547703742981, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6179682612419128, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27524635195732117, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8518228530883789, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6277359127998352, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2737748622894287, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6332215666770935, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6482362747192383, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33636584877967834, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8093819618225098, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5804094672203064, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38402223587036133, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5773451924324036, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.6085790395736694, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4376184940338135, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8391555547714233, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6540367603302002, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3198380470275879, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6614154577255249, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6874393820762634, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.37203362584114075, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8325397968292236, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.722652018070221, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4039769768714905, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7165806293487549, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7301092743873596, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3553113639354706, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.863180935382843, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5404171347618103, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20028123259544373, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5439490675926208, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5406675338745117, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28070175647735596, "metrics/hit_grouping/calorimeter/purity": 0.6727597117424011, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8036342859268188, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.15917161107063293, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11384615302085876, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.16109342873096466, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.12299483269453049, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8057491779327393, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3054048418998718, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09920182079076767, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3042077124118805, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3107101023197174, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24116306006908417, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8531848192214966, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.633296012878418, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2836437523365021, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6385576725006104, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6519371271133423, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3407987952232361, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7544061541557312, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.392259418964386, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5563817024230957, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6055847406387329, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5500597953796387, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8098343014717102, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2725743055343628, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5929117202758789, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6177634596824646, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4139091372489929, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6953166723251343, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4182090163230896, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.48399481177330017, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5389143824577332, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6320390701293945, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.723637580871582, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39774230122566223, "metrics/hit_grouping/tracker/all/object_completeness": 0.5160471200942993, "metrics/hit_grouping/tracker/all/object_purity": 0.5666276812553406, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.587757408618927, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005953815300017595, "metrics/hit_grouping/tracker/background_recall": 0.05857963487505913, "metrics/hit_grouping/tracker/completeness": 0.5265938639640808, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4175630807876587, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7603745460510254, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37102505564689636, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5885266661643982, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6216626763343811, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5373120903968811, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7021801471710205, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41709500551223755, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4870717525482178, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5517255067825317, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6287835240364075, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6648851633071899, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43648868799209595, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3859388828277588, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4602913558483124, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6561488509178162, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7224428653717041, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4039360284805298, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5160483121871948, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.583271861076355, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5810567736625671, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6999907493591309, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4314247667789459, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.49171292781829834, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5726607441902161, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6191744208335876, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7293519377708435, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38375088572502136, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5203243494033813, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5438838601112366, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.590962290763855, "metrics/hit_grouping/tracker/purity": 0.5024685263633728, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.895294189453125, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.059079062193632126, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.17035549879074097, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16934669017791748, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07993049174547195, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7097393870353699, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28719544410705566, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.41856998205184937, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4481724500656128, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5230689644813538, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7018948793411255, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4563485085964203, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5741201043128967, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6340996026992798, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6641284227371216, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.598000705242157, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7313494682312012, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8318513631820679, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.659001350402832, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.11227338761091232, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1281965970993042, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.596073150634766, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3183543086051941, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18791450560092926, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9301939010620117, "metrics/particle/class_1/pid_accuracy": 0.9771972894668579, "metrics/particle/class_2/efficiency": 0.5802109241485596, "metrics/particle/class_2/pid_accuracy": 0.8128383755683899, "metrics/particle/class_3/efficiency": 0.8393580913543701, "metrics/particle/class_3/pid_accuracy": 0.9831821322441101, "metrics/particle/class_4/efficiency": 0.7872954607009888, "metrics/particle/class_4/pid_accuracy": 0.6381418108940125, "metrics/particle/class_5/efficiency": 0.9599999785423279, "metrics/particle/class_5/pid_accuracy": 0.6625000238418579, "metrics/particle/count/bias_mean": -0.8313801884651184, "metrics/particle/count/mae": 2.927734375, "metrics/particle/count/prediction_mean": 69.224609375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05411608889698982, "metrics/particle/event/energy_response_mean": 0.9797926545143127, "metrics/particle/event/met_abs_error": 5.229234218597412, "metrics/particle/event/scalar_pt_relative_abs_error": 0.043054793030023575, "metrics/particle/event/scalar_pt_response_mean": 0.9880258440971375, "metrics/particle/event/vector_pt_closure": 0.04391373321413994, "metrics/particle/matched/delta_eta_abs_mean": 0.012907799333333969, "metrics/particle/matched/delta_phi_abs_mean": 0.012439188547432423, "metrics/particle/matched/delta_r_mean": 0.020166026428341866, "metrics/particle/matched/energy_relative_abs_mean": 0.1188177540898323, "metrics/particle/matched/pid_accuracy": 0.9659205079078674, "metrics/particle/matched/pt_relative_abs_mean": 0.11916410177946091, "metrics/particle/matching/duplicate_fraction": 0.03837146982550621, "metrics/particle/matching/efficiency": 0.8606118559837341, "metrics/particle/matching/f1": 0.865748941898346, "metrics/particle/matching/purity": 0.8709477186203003, "metrics/particle/matching/target_coverage_dr0p05": 0.82230544090271, "metrics/particle/matching/target_coverage_dr0p10": 0.891437292098999, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7106295228004456, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8880452513694763, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020334549248218536, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1170358955860138, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11720624566078186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005579688586294651, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15571314096450806, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005568644031882286, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5646713376045227, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.062371451407670975, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6250001192092896, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9338.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18141789734363556, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7834542989730835, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7449486255645752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6073551774024963, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08557364344596863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.808880805969238, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011737978085875511, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.802911102771759, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0559513233602047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5001487731933594, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10564.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8538771271705627, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8780332803726196, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.6848649978637695, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9902701377868652, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.543647289276123, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.40079769492149353, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.743495464324951, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.197993278503418, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.577855110168457, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0217107106000185, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12761971354484558, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12807908654212952, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014001821167767048, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16830791532993317, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023744292557239532, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2226085662841797, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17146368324756622, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2917468547821045, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2190.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3377108573913574, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8621102571487427, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.3972076177597046, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.2351927757263184, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1351955235004425, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.283843040466309, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.054376162588596344, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1287577152252197, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.28578150272369385, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.396487236022949, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2685.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7249255180358887, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.212344169616699, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.8158183097839355, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.088456630706787, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.005481058731675148, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.020630575716495514, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.692453384399414, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.4476937055587769, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.406339645385742, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019650833681225777, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11821620166301727, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11900003999471664, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010316380299627781, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1578550636768341, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004763668985106051, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19047433137893677, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0343448631465435, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6210694909095764, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18893.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13907180726528168, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7471650838851929, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.598369300365448, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4601321220397949, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08206341415643692, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.635537147521973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026029341388493776, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.313926100730896, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.032854776829481125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3577786684036255, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21130.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8813266754150391, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.2099441289901733, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.854618549346924, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.688817977905273, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1054086685180664, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6177060604095459, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2209126949310303, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9381692409515381, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.7095046043396, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02000899612903595, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11853084713220596, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11910303682088852, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009127724915742874, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15797939896583557, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0037145393434911966, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3796405494213104, "metrics/particle/momentum/all/accepted/pz_bias": -0.05281902849674225, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3528385162353516, "metrics/particle/momentum/all/accepted/samples": 30421.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16759765148162842, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7672932744026184, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7057995200157166, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5659031271934509, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.0872916579246521, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.738190650939941, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009453445672988892, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.605920135974884, "metrics/particle/momentum/all/assigned/pz_bias": -0.05970548093318939, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.4998574256896973, "metrics/particle/momentum/all/assigned/samples": 34379.0, "metrics/particle/momentum/all/efficiency": 0.8594959378242493, "metrics/particle/momentum/all/event/energy_bias": -5.300321578979492, "metrics/particle/momentum/all/event/p3_error_mean": 16.243118286132812, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.136359214782715, "metrics/particle/momentum/all/event/px_bias": 1.6435749530792236, "metrics/particle/momentum/all/event/py_bias": 1.0391342639923096, "metrics/particle/momentum/all/event/pz_abs_mean": 12.70753002166748, "metrics/particle/momentum/all/event/pz_bias": -4.583856105804443, "metrics/particle/momentum/all/event/pz_residual_rms": 22.578411102294922, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021023541688919067, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0917225107550621, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0926414430141449, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011364107020199299, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12572696805000305, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0030267902184277773, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3748883605003357, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05777231603860855, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2305558919906616, "metrics/particle/momentum/class_1/accepted/samples": 15528.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08712399750947952, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21315282583236694, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21356135606765747, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1093444749712944, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03686608001589775, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.0201189517974854, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005223200656473637, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4793892502784729, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04828929901123047, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7837501764297485, "metrics/particle/momentum/class_1/assigned/samples": 16465.0, "metrics/particle/momentum/class_1/efficiency": 0.9305447340011597, "metrics/particle/momentum/class_1/event/energy_bias": 0.06461085379123688, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.429595947265625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.237887382507324, "metrics/particle/momentum/class_1/event/px_bias": 0.7215883135795593, "metrics/particle/momentum/class_1/event/py_bias": 0.4116230607032776, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.715920448303223, "metrics/particle/momentum/class_1/event/pz_bias": -1.495526909828186, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.881441116333008, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02667921595275402, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16975624859333038, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1716882288455963, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.009794466197490692, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21168147027492523, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02500000037252903, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3658356666564941, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.12246496230363846, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5157501697540283, "metrics/particle/momentum/class_2/accepted/samples": 1720.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5230452418327332, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6004887223243713, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6906338930130005, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.38341817259788513, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2325490266084671, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.038027048110962, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04980392009019852, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1621859073638916, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19231590628623962, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.233852386474609, "metrics/particle/momentum/class_2/assigned/samples": 2550.0, "metrics/particle/momentum/class_2/efficiency": 0.5777628421783447, "metrics/particle/momentum/class_2/event/energy_bias": -7.419975280761719, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.467838287353516, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.974410533905029, "metrics/particle/momentum/class_2/event/px_bias": 0.36485400795936584, "metrics/particle/momentum/class_2/event/py_bias": 0.3913253843784332, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.50789737701416, "metrics/particle/momentum/class_2/event/pz_bias": -1.7837904691696167, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.23830795288086, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01785258948802948, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14400553703308105, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14413398504257202, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.005780915264040232, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1826925128698349, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013354280963540077, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23515760898590088, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03882418945431709, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8453592658042908, "metrics/particle/momentum/class_3/accepted/samples": 12730.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.19475972652435303, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3578476905822754, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2221407890319824, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0707719326019287, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.11699333041906357, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.781373977661133, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006668013520538807, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4563256502151489, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04784650728106499, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3963706493377686, "metrics/particle/momentum/class_3/assigned/samples": 14847.0, "metrics/particle/momentum/class_3/efficiency": 0.8371145129203796, "metrics/particle/momentum/class_3/event/energy_bias": 2.592562675476074, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.448371887207031, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.116488933563232, "metrics/particle/momentum/class_3/event/px_bias": 0.4019232392311096, "metrics/particle/momentum/class_3/event/py_bias": 0.02172067016363144, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.408937454223633, "metrics/particle/momentum/class_3/event/pz_bias": -1.455583095550537, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.108882904052734, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022953910753130913, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.13310542702674866, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12746593356132507, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.026731234043836594, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17044970393180847, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01034482754766941, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6712037920951843, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13006505370140076, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.748624563217163, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2600386142730713, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.144148111343384, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2384090423583984, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1041078567504883, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.1643453985452652, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.396879196166992, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019498607143759727, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.149257779121399, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.16825716197490692, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.024868965148926, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.7967032790184021, "metrics/particle/momentum/class_4/event/energy_bias": -0.5470313429832458, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.310074806213379, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5537865161895752, "metrics/particle/momentum/class_4/event/px_bias": 0.18856197595596313, "metrics/particle/momentum/class_4/event/py_bias": -0.02473919838666916, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4460619688034058, "metrics/particle/momentum/class_4/event/pz_bias": 0.2612139582633972, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.607729911804199, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01589379832148552, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11626358330249786, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.11504693329334259, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01975875161588192, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.14898544549942017, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.244020938873291, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.27821114659309387, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.4495582580566406, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05459528788924217, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3117240369319916, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24408507347106934, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.151812344789505, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03164556995034218, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1394037008285522, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.037974681705236435, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4971972703933716, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.02313702553510666, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.1096959114074707, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": 0.009511677548289299, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.516154408454895, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1644877195358276, "metrics/particle/momentum/class_5/event/px_bias": -0.033352598547935486, "metrics/particle/momentum/class_5/event/py_bias": 0.23920436203479767, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7699670791625977, "metrics/particle/momentum/class_5/event/pz_bias": -0.11016945540904999, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.4311983585357666, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2892570495605469, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.290802538394928, "metrics/set_slot/angular_recall_all_dr0p10": 0.9657546877861023, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8757318258285522, "metrics/set_slot/soft_count_bias_mean": 0.3670296370983124}, "step_train_time": 5.595397233963013, "step_valid_time": 45662.63374376297, "step_total_time": 45668.22914147377, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.992686713563135, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10609224419282881, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2888, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9593657086223984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9638234317955526, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9615894039735099, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036176568204447396, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9540799471423852, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585131098572851, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9562913907284768, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9593657086223984, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9910416168884095, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10201531350914483, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2161, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9553492484526968, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9566179725542275, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9559831895598319, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043382027445772464, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526967285587975, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539619300575476, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533289095332891, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9553492484526968, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9888943195926764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09902622178662757, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1210, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9394409937888198, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9475332811276429, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434697855750487, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052466718872357085, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9467501957713391, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9426900584795321, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9394409937888198} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_45000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..7373fecc084b29e3e92df811963e8b1fc2c96701 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131893121091529, "train": {"Classification_binary": 0.1410052329301834, "Cardinality": 0.11888370662927628, "Classification": 0.06338877230882645, "Regression_pt": 0.09153462946414948, "Regression_eta": 0.001220355392433703, "Regression_sin_phi": 0.0006371161434799433, "Regression_cos_phi": 0.0006425146129913628, "Regression_energy": 0.09169143438339233, "HitGrouping": 0.14105607569217682, "Auxiliary": 0.17916233837604523, "Total": 0.8448037505149841, "Total_unweighted": 0.3901200552354567}, "train_group": {}, "valid": {"Classification_binary": 0.1252858837445577, "Cardinality": 0.0952024757862091, "Classification": 0.06209843854109446, "Regression_pt": 0.08611332376797994, "Regression_eta": 0.0010990970768034458, "Regression_sin_phi": 0.0006455297116190195, "Regression_cos_phi": 0.0006307674727092186, "Regression_energy": 0.08580302198727925, "HitGrouping": 0.14018417398134866, "Auxiliary": 0.16614223519961038, "Total": 0.7810389200846354, "Total_unweighted": 0.361676062302043, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8489601612091064, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6168045997619629, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28119391202926636, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6226306557655334, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6324089169502258, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33068856596946716, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7981293201446533, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5612948536872864, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.29549306631088257, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.569651186466217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5867564082145691, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.40596407651901245, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8583399057388306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5998964905738831, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.26221466064453125, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.60347980260849, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6114840507507324, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.311693012714386, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.849920928478241, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.6018750071525574, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27130699157714844, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6066081523895264, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6159995794296265, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32626140117645264, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026786718517541885, "metrics/hit_grouping/calorimeter/background_recall": 0.6507794260978699, "metrics/hit_grouping/calorimeter/completeness": 0.6781755685806274, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6881792545318604, "metrics/hit_grouping/calorimeter/energy_purity": 0.6804490685462952, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30616071820259094, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8665086627006531, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6056997776031494, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22054344415664673, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6139733195304871, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6126245856285095, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2769184708595276, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8534155488014221, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6236163973808289, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2771712839603424, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.628037691116333, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6375098824501038, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3287239074707031, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8111682534217834, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5711226463317871, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37264567613601685, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5686870217323303, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.6007276177406311, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.42813804745674133, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8405349850654602, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6508811116218567, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3190283477306366, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6580029129981995, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6814697980880737, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36250388622283936, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8300541043281555, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7153719067573547, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4055468440055847, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7106637954711914, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7249389290809631, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3642072081565857, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.862553596496582, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5346668362617493, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20282576978206635, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.538040280342102, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5316745638847351, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28244274854660034, "metrics/hit_grouping/calorimeter/purity": 0.6688719391822815, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8048498630523682, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.15776175260543823, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10307691991329193, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.15633699297904968, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.1262006014585495, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03692307695746422, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8058463335037231, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3030311167240143, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10718358308076859, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.30295732617378235, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2985675036907196, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24230331182479858, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8534165620803833, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6285291314125061, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28433236479759216, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6336700916290283, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6446965932846069, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33725735545158386, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7583908438682556, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3829946219921112, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5607748031616211, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6111177802085876, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5391512513160706, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8091800808906555, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2708917558193207, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5844786763191223, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6128796339035034, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.39764443039894104, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6993977427482605, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.41442859172821045, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4875066876411438, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.538364827632904, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.629203736782074, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7272894382476807, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39240506291389465, "metrics/hit_grouping/tracker/all/object_completeness": 0.5188315510749817, "metrics/hit_grouping/tracker/all/object_purity": 0.5676356554031372, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5812394022941589, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005624403245747089, "metrics/hit_grouping/tracker/background_recall": 0.053011905401945114, "metrics/hit_grouping/tracker/completeness": 0.5322920680046082, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4100078344345093, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7648008465766907, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.371753990650177, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5932387113571167, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6207922101020813, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5281093120574951, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.705091118812561, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4115743339061737, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.490826815366745, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.556357741355896, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6242147088050842, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6676047444343567, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41788026690483093, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.383413165807724, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4616183042526245, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6535193920135498, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7258713245391846, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.39459022879600525, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5204542875289917, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5890688896179199, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5767433643341064, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7039166688919067, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.41411450505256653, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48613885045051575, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5713222026824951, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6125166416168213, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7332466840744019, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38574105501174927, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5224663615226746, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5390655398368835, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5815967917442322, "metrics/hit_grouping/tracker/purity": 0.507737398147583, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8901479840278625, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05212858319282532, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16077962517738342, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1660711169242859, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08340573310852051, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7161007523536682, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2835666239261627, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40836936235427856, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4379791021347046, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.4971487820148468, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7064336538314819, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4510359466075897, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5800628066062927, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6369650959968567, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6584617495536804, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.6002498865127563, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7407430410385132, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.843224048614502, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6592733263969421, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.11452259123325348, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.12846854329109192, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.022274017333984, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3327813148498535, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20234151184558868, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9361693859100342, "metrics/particle/class_1/pid_accuracy": 0.978040337562561, "metrics/particle/class_2/efficiency": 0.5705631375312805, "metrics/particle/class_2/pid_accuracy": 0.8077074289321899, "metrics/particle/class_3/efficiency": 0.819041907787323, "metrics/particle/class_3/pid_accuracy": 0.9808528423309326, "metrics/particle/class_4/efficiency": 0.790182888507843, "metrics/particle/class_4/pid_accuracy": 0.6321558952331543, "metrics/particle/class_5/efficiency": 0.9539999961853027, "metrics/particle/class_5/pid_accuracy": 0.6645702123641968, "metrics/particle/count/bias_mean": -1.1471353769302368, "metrics/particle/count/mae": 3.0026042461395264, "metrics/particle/count/prediction_mean": 68.90885162353516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05369439348578453, "metrics/particle/event/energy_response_mean": 0.998461902141571, "metrics/particle/event/met_abs_error": 4.993654251098633, "metrics/particle/event/scalar_pt_relative_abs_error": 0.042845506221055984, "metrics/particle/event/scalar_pt_response_mean": 1.0093799829483032, "metrics/particle/event/vector_pt_closure": 0.042893555015325546, "metrics/particle/matched/delta_eta_abs_mean": 0.012278081849217415, "metrics/particle/matched/delta_phi_abs_mean": 0.012343103066086769, "metrics/particle/matched/delta_r_mean": 0.019554466009140015, "metrics/particle/matched/energy_relative_abs_mean": 0.11787664145231247, "metrics/particle/matched/pid_accuracy": 0.965061604976654, "metrics/particle/matched/pt_relative_abs_mean": 0.11820114403963089, "metrics/particle/matching/duplicate_fraction": 0.03956766426563263, "metrics/particle/matching/efficiency": 0.8538185358047485, "metrics/particle/matching/f1": 0.8608667254447937, "metrics/particle/matching/purity": 0.8680322170257568, "metrics/particle/matching/target_coverage_dr0p05": 0.8218872547149658, "metrics/particle/matching/target_coverage_dr0p10": 0.8898946046829224, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7046261429786682, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8804248571395874, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019574284553527832, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11577490717172623, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11648082733154297, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015706533566117287, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15583492815494537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004759844392538071, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5591937899589539, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.025364957749843597, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6015359163284302, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9244.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17776364088058472, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7917295098304749, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8539195656776428, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7259639501571655, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09548215568065643, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.73824405670166, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010914958082139492, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8020399808883667, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0040360395796597, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4880380630493164, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10536.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8452816605567932, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.4092288017272949, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.459056854248047, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.903663396835327, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.07075788080692291, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.33134356141090393, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.5406060218811035, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0173028707504272, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.36378002166748, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021010255441069603, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12561246752738953, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12511785328388214, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.027820635586977005, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.166755810379982, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021668972447514534, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2237766981124878, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19002602994441986, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3138465881347656, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2169.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3235335648059845, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7370989322662354, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.3152471780776978, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.185249924659729, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1385020762681961, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 9.883336067199707, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05193827673792839, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.126488208770752, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19960127770900726, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.5483903884887695, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2657.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7179741859436035, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2797598838806152, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.5650506019592285, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0263285636901855, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12706570327281952, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05416583642363548, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.442412853240967, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.8441593647003174, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.569372177124023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019013844430446625, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11645868420600891, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11648079752922058, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019171589985489845, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15641668438911438, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003735524951480329, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19006548821926117, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021247737109661102, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6188942193984985, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18739.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13775913417339325, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8265841007232666, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6684630513191223, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5448869466781616, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08687615394592285, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.033166885375977, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024645717348903418, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2976219952106476, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02432996965944767, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.176224946975708, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21099.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8741428256034851, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.358370065689087, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.788673400878906, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.725245952606201, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.10883452743291855, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.2530196905136108, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0500266551971436, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.7006567716598511, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.038107395172119, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01932927779853344, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11690753698348999, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11710211634635925, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01873144693672657, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1570058912038803, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032501989044249058, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.37759339809417725, "metrics/particle/momentum/all/accepted/pz_bias": -0.03465115278959274, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.346983790397644, "metrics/particle/momentum/all/accepted/samples": 30152.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16444435715675354, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8089417219161987, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.775557279586792, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6501380801200867, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09352035820484161, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.322587966918945, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008894202299416065, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5943047404289246, "metrics/particle/momentum/all/assigned/pz_bias": -0.03167509287595749, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.464893102645874, "metrics/particle/momentum/all/assigned/samples": 34292.0, "metrics/particle/momentum/all/efficiency": 0.851895809173584, "metrics/particle/momentum/all/event/energy_bias": 0.48783934116363525, "metrics/particle/momentum/all/event/p3_error_mean": 15.368874549865723, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.052768230438232, "metrics/particle/momentum/all/event/px_bias": -0.16514232754707336, "metrics/particle/momentum/all/event/py_bias": 1.5301975011825562, "metrics/particle/momentum/all/event/pz_abs_mean": 11.716623306274414, "metrics/particle/momentum/all/event/pz_bias": -3.5621190071105957, "metrics/particle/momentum/all/event/pz_residual_rms": 20.315452575683594, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.020293982699513435, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.08917272090911865, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.08982094377279282, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004896274767816067, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12355034053325653, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002688688226044178, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.36611831188201904, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.050592124462127686, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2060428857803345, "metrics/particle/momentum/class_1/accepted/samples": 15621.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07595037668943405, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.17160150408744812, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.18083533644676208, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.0778489038348198, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03229527175426483, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7435063123703003, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00473502092063427, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.44970905780792236, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.031996749341487885, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6163849830627441, "metrics/particle/momentum/class_1/assigned/samples": 16473.0, "metrics/particle/momentum/class_1/efficiency": 0.936117947101593, "metrics/particle/momentum/class_1/event/energy_bias": 2.3841183185577393, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.6050443649292, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.565092086791992, "metrics/particle/momentum/class_1/event/px_bias": -0.6239899396896362, "metrics/particle/momentum/class_1/event/py_bias": 0.7049652338027954, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.704068183898926, "metrics/particle/momentum/class_1/event/pz_bias": -2.8888025283813477, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.27863883972168, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026835303753614426, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16852276027202606, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.16897951066493988, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02777927555143833, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.20856639742851257, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02175191044807434, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3422309160232544, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08461561799049377, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3503663539886475, "metrics/particle/momentum/class_2/accepted/samples": 1701.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.47621339559555054, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6005324721336365, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7266005277633667, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4507492482662201, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2407185584306717, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.054986476898193, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04391217604279518, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.0968337059020996, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.005237536504864693, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.303529739379883, "metrics/particle/momentum/class_2/assigned/samples": 2505.0, "metrics/particle/momentum/class_2/efficiency": 0.5713805556297302, "metrics/particle/momentum/class_2/event/energy_bias": -5.5069966316223145, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.917927742004395, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.4824113845825195, "metrics/particle/momentum/class_2/event/px_bias": 0.6401083469390869, "metrics/particle/momentum/class_2/event/py_bias": 0.288809597492218, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.584269523620605, "metrics/particle/momentum/class_2/event/pz_bias": -0.49097707867622375, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.89698028564453, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017088910564780235, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14433327317237854, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14404939115047455, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03393254801630974, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18380603194236755, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013721849536523223, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24123668670654297, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009576648473739624, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9477682709693909, "metrics/particle/momentum/class_3/accepted/samples": 12389.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20941580832004547, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.505355715751648, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4079407453536987, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2822822332382202, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13607189059257507, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.595296859741211, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007499493192881346, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4841998517513275, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.031538043171167374, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4711341857910156, "metrics/particle/momentum/class_3/assigned/samples": 14801.0, "metrics/particle/momentum/class_3/efficiency": 0.8146905899047852, "metrics/particle/momentum/class_3/event/energy_bias": 3.60017728805542, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.632363319396973, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.261855125427246, "metrics/particle/momentum/class_3/event/px_bias": -0.07828609645366669, "metrics/particle/momentum/class_3/event/py_bias": 0.45757120847702026, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.5022807121276855, "metrics/particle/momentum/class_3/event/pz_bias": -0.3209095597267151, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.257070541381836, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02220398560166359, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1337166726589203, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12943047285079956, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.048716168850660324, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.17175425589084625, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006896551698446274, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6242982745170593, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.15419068932533264, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.616624116897583, "metrics/particle/momentum/class_4/accepted/samples": 290.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.26053738594055176, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.1146109104156494, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.633618116378784, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.5173325538635254, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14647887647151947, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.78519058227539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8802569508552551, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.14158320426940918, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0074591636657715, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.7967032790184021, "metrics/particle/momentum/class_4/event/energy_bias": -0.354358971118927, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3216235637664795, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5899735689163208, "metrics/particle/momentum/class_4/event/px_bias": -0.00043255463242530823, "metrics/particle/momentum/class_4/event/py_bias": -0.0483696386218071, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.408333420753479, "metrics/particle/momentum/class_4/event/pz_bias": 0.2682294249534607, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.654355049133301, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013268781825900078, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12217763066291809, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1203528419137001, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.043280113488435745, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.15687893331050873, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4119222164154053, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.24265459179878235, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.580141544342041, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019159747287631035, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.14335528016090393, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.14245925843715668, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.06513971090316772, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.037974681705236435, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.20755033195018768, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.006329114083200693, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5198729038238525, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.18318453431129456, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.8198399543762207, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9496855139732361, "metrics/particle/momentum/class_5/event/energy_bias": 0.3648996651172638, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6792031526565552, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2754168510437012, "metrics/particle/momentum/class_5/event/px_bias": -0.10254200547933578, "metrics/particle/momentum/class_5/event/py_bias": 0.12722106277942657, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8531810641288757, "metrics/particle/momentum/class_5/event/pz_bias": -0.12965907156467438, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.6362133026123047, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2885674238204956, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2899690270423889, "metrics/set_slot/angular_recall_all_dr0p10": 0.9659870266914368, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8755459785461426, "metrics/set_slot/soft_count_bias_mean": -0.2567152678966522}, "step_train_time": 5.606628656387329, "step_valid_time": 48662.29401063919, "step_total_time": 48667.900639772415, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0128478107526544, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10163355003125363, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2917, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9636603898249092, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9617540389053743, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9627062706270627, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03824596109462578, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.958374628344896, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564787339268052, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574257425742574, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9636603898249092, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0087482644048658, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10601822364923763, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2263, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2166, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2159, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9575596816976127, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9571365444100751, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573480662983426, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04286345558992488, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9544650751547303, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540433053468846, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9542541436464088, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9575596816976127, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0090818618908282, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10053026999265924, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1291, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1218, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1215, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9456521739130435, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9434546862896979, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.944552151996898, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056545313710302095, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9433229813664596, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.941130906274206, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9422256688639007, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9456521739130435} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_46000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..09e28cd84aea98f32cb8e0e00552bc03953dcd08 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.179268523782452, "train": {"Classification_binary": 0.13010095059871674, "Cardinality": 0.09644520282745361, "Classification": 0.05489472672343254, "Regression_pt": 0.0839814841747284, "Regression_eta": 0.0009315575589425862, "Regression_sin_phi": 0.0005292210844345391, "Regression_cos_phi": 0.0005324753001332283, "Regression_energy": 0.08370229601860046, "HitGrouping": 0.1373937427997589, "Auxiliary": 0.17043668031692505, "Total": 0.7613638639450073, "Total_unweighted": 0.3546727114589885}, "train_group": {}, "valid": {"Classification_binary": 0.12493786215782166, "Cardinality": 0.09483915567398071, "Classification": 0.06256057322025299, "Regression_pt": 0.08790218830108643, "Regression_eta": 0.0011074626818299294, "Regression_sin_phi": 0.0006344142214705547, "Regression_cos_phi": 0.0006431109892825285, "Regression_energy": 0.08724388480186462, "HitGrouping": 0.1416678031285604, "Auxiliary": 0.16739789644877115, "Total": 0.7854092915852865, "Total_unweighted": 0.3650294963736087, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8500385880470276, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6110366582870483, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.27571550011634827, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6172289848327637, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6265198588371277, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3275715410709381, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.799872875213623, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5521780252456665, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2853270173072815, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5601773262023926, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5814321637153625, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4130803048610687, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8619903922080994, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5958751440048218, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.26479703187942505, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.6000766158103943, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.604381263256073, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3059084713459015, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8525726199150085, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.596834659576416, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2701519727706909, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.602017343044281, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6094468832015991, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.322492390871048, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026600267738103867, "metrics/hit_grouping/calorimeter/background_recall": 0.6498479247093201, "metrics/hit_grouping/calorimeter/completeness": 0.6729092597961426, "metrics/hit_grouping/calorimeter/energy_completeness": 0.684178352355957, "metrics/hit_grouping/calorimeter/energy_purity": 0.6764141917228699, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30428749322891235, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8687226176261902, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.601168155670166, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21660196781158447, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6093320846557617, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6086658835411072, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2763810157775879, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8567794561386108, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6187880039215088, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2771712839603424, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6241863965988159, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.631239652633667, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3216885030269623, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8140046000480652, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5647842884063721, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3761850595474243, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5634323358535767, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5883899927139282, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4209328889846802, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8427659273147583, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6474907398223877, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31803175806999207, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6555812954902649, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6767632365226746, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3578324615955353, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8361373543739319, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7169787287712097, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.41234955191612244, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7135193347930908, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7212424874305725, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3474620580673218, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8652186393737793, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5270019173622131, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20048211514949799, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5301645398139954, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5227718353271484, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28130441904067993, "metrics/hit_grouping/calorimeter/purity": 0.6635507345199585, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.810448169708252, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1505691558122635, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10615384578704834, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.14984799921512604, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.11504805088043213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.035384614020586014, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8176999688148499, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2861207127571106, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09977194666862488, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28606054186820984, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28818467259407043, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24002280831336975, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8554762601852417, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6242173314094543, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2834469974040985, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6298274993896484, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.638462245464325, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.333355188369751, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7563866972923279, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40047818422317505, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5595817565917969, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6001951098442078, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5473700165748596, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8122925162315369, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28379136323928833, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.594131350517273, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.61388099193573, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.39484015107154846, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.7007222175598145, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.41395604610443115, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.48461097478866577, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5289623737335205, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6310151815414429, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7277123928070068, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3987341821193695, "metrics/hit_grouping/tracker/all/object_completeness": 0.5175307989120483, "metrics/hit_grouping/tracker/all/object_purity": 0.5586288571357727, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.584687352180481, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005542050115764141, "metrics/hit_grouping/tracker/background_recall": 0.05271574854850769, "metrics/hit_grouping/tracker/completeness": 0.5310829877853394, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.42097416520118713, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7616374492645264, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3787699341773987, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5885714888572693, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6139558553695679, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.537858784198761, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7088696956634521, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4090995490550995, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.49159494042396545, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5505176186561584, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6185036897659302, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6724240183830261, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4320388436317444, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.38738712668418884, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4444286823272705, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6527103781700134, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7275094985961914, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.40132996439933777, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.51882404088974, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5792555212974548, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5789000988006592, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7175449132919312, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4474034607410431, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.5055996179580688, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5438622236251831, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5992010831832886, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7297646403312683, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38679465651512146, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5179440379142761, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5343542695045471, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5896745324134827, "metrics/hit_grouping/tracker/purity": 0.5065421462059021, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8955637216567993, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05777584761381149, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16717560589313507, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1636122316122055, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07732406258583069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7102720737457275, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27630895376205444, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.408291220664978, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4272136986255646, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5318818092346191, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7068901062011719, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.45900478959083557, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5775769352912903, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6272687315940857, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6596422791481018, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5953695774078369, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.732694149017334, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8338431715965271, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.661255955696106, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10964225232601166, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.13045115768909454, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.566139221191406, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3096826374530792, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1792427897453308, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9345270991325378, "metrics/particle/class_1/pid_accuracy": 0.9789121747016907, "metrics/particle/class_2/efficiency": 0.5836885571479797, "metrics/particle/class_2/pid_accuracy": 0.810878336429596, "metrics/particle/class_3/efficiency": 0.8261858224868774, "metrics/particle/class_3/pid_accuracy": 0.9833294153213501, "metrics/particle/class_4/efficiency": 0.7949951887130737, "metrics/particle/class_4/pid_accuracy": 0.6464890837669373, "metrics/particle/class_5/efficiency": 0.9739999771118164, "metrics/particle/class_5/pid_accuracy": 0.6673511266708374, "metrics/particle/count/bias_mean": -1.1217447519302368, "metrics/particle/count/mae": 2.9524738788604736, "metrics/particle/count/prediction_mean": 68.93424224853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05267302691936493, "metrics/particle/event/energy_response_mean": 0.9838560223579407, "metrics/particle/event/met_abs_error": 5.11205530166626, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040700312703847885, "metrics/particle/event/scalar_pt_response_mean": 0.9955123066902161, "metrics/particle/event/vector_pt_closure": 0.04288403317332268, "metrics/particle/matched/delta_eta_abs_mean": 0.011628227308392525, "metrics/particle/matched/delta_phi_abs_mean": 0.012373349629342556, "metrics/particle/matched/delta_r_mean": 0.019107811152935028, "metrics/particle/matched/energy_relative_abs_mean": 0.11871708929538727, "metrics/particle/matched/pid_accuracy": 0.9666587114334106, "metrics/particle/matched/pt_relative_abs_mean": 0.11941255629062653, "metrics/particle/matching/duplicate_fraction": 0.037446994334459305, "metrics/particle/matching/efficiency": 0.857368528842926, "metrics/particle/matching/f1": 0.8642880916595459, "metrics/particle/matching/purity": 0.8713202476501465, "metrics/particle/matching/target_coverage_dr0p05": 0.8240339756011963, "metrics/particle/matching/target_coverage_dr0p10": 0.8902849555015564, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7053695917129517, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8843744993209839, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01891191676259041, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11710269004106522, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11708424985408783, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010378275066614151, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15653209388256073, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006024744361639023, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5707001686096191, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0519605353474617, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6509788036346436, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9295.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17581072449684143, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9708849787712097, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1857082843780518, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0503262281417847, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.0911153107881546, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 36.53337860107422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01172022707760334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8283076882362366, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05289556458592415, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7186620235443115, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10580.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8499451279640198, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.101620078086853, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.150163650512695, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2041103839874268, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3439762592315674, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.4718736410140991, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.107706069946289, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.189699172973633, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.362741470336914, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02046061120927334, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1275610625743866, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1274385154247284, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026088016107678413, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16863465309143066, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02253909781575203, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2261346578598022, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19642767310142517, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2489449977874756, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2174.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.32833173871040344, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0147651433944702, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.961525321006775, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8167510032653809, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.14317840337753296, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.158769607543945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05359819903969765, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1502368450164795, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3303896486759186, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.4596662521362305, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2668.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7196292877197266, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.5408053398132324, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.82917594909668, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1769522428512573, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02531081810593605, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.036177631467580795, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.679790496826172, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3866745233535767, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.573993682861328, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01875169388949871, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11780833452939987, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1187070980668068, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017220644280314445, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15851494669914246, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004776056157425046, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19264817237854004, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.01525494921952486, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6533548831939697, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18844.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14232948422431946, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7512863278388977, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5363743305206299, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4044964909553528, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08267790824174881, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.91365909576416, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028901733458042145, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3086790144443512, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.009155366569757462, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.377707600593567, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21106.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8790408968925476, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.230195552110672, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.877379894256592, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.730179786682129, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6491159796714783, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.18886327743530273, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.142850875854492, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.17076200246810913, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.5471930503845215, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018923385068774223, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11829142272472382, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1188356801867485, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015758495777845383, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15865960717201233, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003760762745514512, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3826918303966522, "metrics/particle/momentum/all/accepted/pz_bias": -0.0395035482943058, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3631582260131836, "metrics/particle/momentum/all/accepted/samples": 30313.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16708602011203766, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8393783569335938, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8470297455787659, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7130711078643799, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.08997496962547302, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.039438247680664, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009547650814056396, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6117280125617981, "metrics/particle/momentum/all/assigned/pz_bias": -0.04757373034954071, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5851471424102783, "metrics/particle/momentum/all/assigned/samples": 34354.0, "metrics/particle/momentum/all/efficiency": 0.8564445972442627, "metrics/particle/momentum/all/event/energy_bias": -3.8726210594177246, "metrics/particle/momentum/all/event/p3_error_mean": 15.478107452392578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.286532402038574, "metrics/particle/momentum/all/event/px_bias": 0.9677814841270447, "metrics/particle/momentum/all/event/py_bias": 0.6245592832565308, "metrics/particle/momentum/all/event/pz_abs_mean": 11.757027626037598, "metrics/particle/momentum/all/event/pz_bias": -3.7471354007720947, "metrics/particle/momentum/all/event/pz_residual_rms": 20.982139587402344, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019981594756245613, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09011053293943405, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09082039445638657, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009217837825417519, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12395917624235153, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003079094225540757, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3666103482246399, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04739692434668541, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2032147645950317, "metrics/particle/momentum/class_1/accepted/samples": 15589.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08482051640748978, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1783829629421234, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.18731051683425903, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08422311395406723, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03492390736937523, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.4912374019622803, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00539622874930501, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4652377963066101, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.040128786116838455, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.694850206375122, "metrics/particle/momentum/class_1/assigned/samples": 16493.0, "metrics/particle/momentum/class_1/efficiency": 0.9342002868652344, "metrics/particle/momentum/class_1/event/energy_bias": 2.1332061290740967, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.883980751037598, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.1776862144470215, "metrics/particle/momentum/class_1/event/px_bias": 0.5359688997268677, "metrics/particle/momentum/class_1/event/py_bias": -0.014389276504516602, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.105406761169434, "metrics/particle/momentum/class_1/event/pz_bias": -1.317352056503296, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.75454330444336, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026730788871645927, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16845287382602692, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1700488179922104, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.008450114168226719, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2105819582939148, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02300172485411167, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.360417127609253, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11257702112197876, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.456144332885742, "metrics/particle/momentum/class_2/accepted/samples": 1739.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.49197277426719666, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6479452252388, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7555930614471436, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.44346433877944946, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.22939778864383698, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.063249111175537, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04675118997693062, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.202268123626709, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13431324064731598, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.747562885284424, "metrics/particle/momentum/class_2/assigned/samples": 2524.0, "metrics/particle/momentum/class_2/efficiency": 0.5841451287269592, "metrics/particle/momentum/class_2/event/energy_bias": -8.078756332397461, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.48591423034668, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.9163899421691895, "metrics/particle/momentum/class_2/event/px_bias": 0.22182230651378632, "metrics/particle/momentum/class_2/event/py_bias": 0.348207950592041, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.543696403503418, "metrics/particle/momentum/class_2/event/pz_bias": -1.3951685428619385, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.29189682006836, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016488851979374886, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14562471210956573, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14590221643447876, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023835420608520508, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1856822520494461, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015951507957652211, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24596452713012695, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.022740835323929787, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9160172343254089, "metrics/particle/momentum/class_3/accepted/samples": 12538.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20330810546875, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5449591875076294, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5565677881240845, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.417614459991455, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12721753120422363, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 33.06166076660156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007487352471798658, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.482536643743515, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.040722694247961044, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4706056118011475, "metrics/particle/momentum/class_3/assigned/samples": 14825.0, "metrics/particle/momentum/class_3/efficiency": 0.8244886994361877, "metrics/particle/momentum/class_3/event/energy_bias": 2.415997266769409, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.398083686828613, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.231295108795166, "metrics/particle/momentum/class_3/event/px_bias": 0.19675061106681824, "metrics/particle/momentum/class_3/event/py_bias": 0.09867721796035767, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.252408504486084, "metrics/particle/momentum/class_3/event/pz_bias": -0.6738169193267822, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.734062194824219, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02308952994644642, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14478622376918793, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14162079989910126, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05867232382297516, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1834230124950409, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006849315017461777, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7302017211914062, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.27358946204185486, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0632877349853516, "metrics/particle/momentum/class_4/accepted/samples": 292.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.22637738287448883, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.651876926422119, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.773148775100708, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.6679701805114746, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13276836276054382, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.7747688293457, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.024869680404663, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04441216215491295, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.1184258460998535, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.8021978139877319, "metrics/particle/momentum/class_4/event/energy_bias": -0.3072614371776581, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3251378536224365, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.515610694885254, "metrics/particle/momentum/class_4/event/px_bias": 0.09230268001556396, "metrics/particle/momentum/class_4/event/py_bias": -0.035653531551361084, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.482923984527588, "metrics/particle/momentum/class_4/event/pz_bias": -0.21759162843227386, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.59677267074585, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013982442207634449, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12886740267276764, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12952664494514465, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0213856790214777, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16661964356899261, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025806451216340065, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4358775615692139, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.37156471610069275, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.8092243671417236, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03296348452568054, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3907378613948822, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2823878526687622, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.17630019783973694, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.018987340852618217, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.8581637144088745, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.037974681705236435, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6911758184432983, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.08899595588445663, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.846196413040161, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9748427867889404, "metrics/particle/momentum/class_5/event/energy_bias": -0.03580624610185623, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.692877173423767, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2377347946166992, "metrics/particle/momentum/class_5/event/px_bias": -0.07906293123960495, "metrics/particle/momentum/class_5/event/py_bias": 0.22771692276000977, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9154501557350159, "metrics/particle/momentum/class_5/event/pz_bias": -0.14320649206638336, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.1419551372528076, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.28655022382736206, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2881905734539032, "metrics/set_slot/angular_recall_all_dr0p10": 0.9665632247924805, "metrics/set_slot/angular_recall_selected_dr0p10": 0.874895453453064, "metrics/set_slot/soft_count_bias_mean": -0.12471997737884521}, "step_train_time": 7.191590785980225, "step_valid_time": 51720.19275665283, "step_total_time": 51727.38434767723, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044561231087619, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10521786915534426, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2912, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9620085893624051, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9594728171334431, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607390300230947, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040527182866556835, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9573835480673935, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548599670510708, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9561200923787528, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9620085893624051, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000016788256553, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10500539341024362, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9531388152077808, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9599287622439893, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565217391304348, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04007123775601069, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9509283819628647, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9577025823686554, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9543034605146407, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9531388152077808, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9978151651493112, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09913077601420173, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1208, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1204, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.937888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.937888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.937888198757764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.062111801242236024, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9347826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9347826086956522, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.937888198757764} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_47000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..9dcea28640483919e14325d259aa706ca5efc829 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226643926473375, "train": {"Classification_binary": 0.1381181925535202, "Cardinality": 0.10121861100196838, "Classification": 0.05802253261208534, "Regression_pt": 0.08213919401168823, "Regression_eta": 0.0012443533632904291, "Regression_sin_phi": 0.0006673826719634235, "Regression_cos_phi": 0.0005830281879752874, "Regression_energy": 0.08237986266613007, "HitGrouping": 0.13978858292102814, "Auxiliary": 0.17314469814300537, "Total": 0.8022825121879578, "Total_unweighted": 0.363154546066653}, "train_group": {}, "valid": {"Classification_binary": 0.12531701723734537, "Cardinality": 0.0913364589214325, "Classification": 0.060221840937932335, "Regression_pt": 0.08289400239785512, "Regression_eta": 0.001122625855108102, "Regression_sin_phi": 0.000630593936269482, "Regression_cos_phi": 0.0006240520548696319, "Regression_energy": 0.0824940452973048, "HitGrouping": 0.1409486730893453, "Auxiliary": 0.16510509451230368, "Total": 0.7714231014251709, "Total_unweighted": 0.3533041777166848, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8451564311981201, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6109272241592407, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2764711380004883, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6163824796676636, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6259040832519531, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3350335359573364, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.798500120639801, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5644506216049194, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30294814705848694, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5728397965431213, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5910380482673645, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4130803048610687, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8544149398803711, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5901758074760437, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2578762471675873, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5941599011421204, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.604070246219635, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31391385197639465, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8464202880859375, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5945460200309753, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2679027318954468, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5993986129760742, "metrics/hit_grouping/calorimeter/all/object_purity": 0.609927237033844, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3296048641204834, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026966482400894165, "metrics/hit_grouping/calorimeter/background_recall": 0.6536973714828491, "metrics/hit_grouping/calorimeter/completeness": 0.6763659715652466, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6884918212890625, "metrics/hit_grouping/calorimeter/energy_purity": 0.6807263493537903, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30003678798675537, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8606357574462891, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5957329273223877, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2177366316318512, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.603985071182251, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.602010190486908, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2831293046474457, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8529660701751709, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6191708445549011, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27353227138519287, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.623638927936554, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6381005644798279, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3307860195636749, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.80950927734375, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5663723349571228, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3682214617729187, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5644299983978271, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5973259210586548, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.42674756050109863, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8398304581642151, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6512543559074402, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31392088532447815, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6584907174110413, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6851798295974731, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36661475896835327, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8328649401664734, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7122671604156494, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.39351126551628113, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7095614671707153, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7243983149528503, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3579277992248535, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.85523921251297, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5185170769691467, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20235703885555267, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5217740535736084, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5143778920173645, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28619256615638733, "metrics/hit_grouping/calorimeter/purity": 0.6672104001045227, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7741684913635254, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.14126813411712646, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10307691991329193, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.14163735508918762, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.10096732527017593, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05230769142508507, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.767017126083374, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.25656744837760925, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1009121984243393, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.25575578212738037, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.24856515228748322, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2559863030910492, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8525271415710449, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6236464381217957, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28102046251296997, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.628920316696167, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.641559362411499, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3397494852542877, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7567577958106995, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3921099901199341, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5626509785652161, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6112257242202759, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.538852334022522, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8089271783828735, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2849130630493164, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5952601432800293, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.620616614818573, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41166573762893677, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6968013048171997, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.41316846013069153, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4889097809791565, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5472748875617981, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6307789087295532, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7251949906349182, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3957113027572632, "metrics/hit_grouping/tracker/all/object_completeness": 0.5211740136146545, "metrics/hit_grouping/tracker/all/object_purity": 0.5736648440361023, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5832703709602356, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.006048368755728006, "metrics/hit_grouping/tracker/background_recall": 0.06213350594043732, "metrics/hit_grouping/tracker/completeness": 0.5329127311706543, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4112338721752167, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7635510563850403, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3763097822666168, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5943306684494019, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6268417835235596, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5308428406715393, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7027906775474548, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4085284471511841, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.49304476380348206, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5644243359565735, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.626879870891571, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6638541221618652, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42516180872917175, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.388663649559021, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4654374122619629, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6533171534538269, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.724301278591156, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.39485982060432434, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5223705768585205, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5960073471069336, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5765635967254639, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6953496932983398, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.416777640581131, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.48603177070617676, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5686152577400208, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6311584711074829, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7316069602966309, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3931163549423218, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.525794506072998, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5454462766647339, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5835869908332825, "metrics/hit_grouping/tracker/purity": 0.5085462331771851, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8928572535514832, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.059947870671749115, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16179540753364563, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16238662600517273, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07819287478923798, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7104648947715759, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28045618534088135, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.41670480370521545, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4404209554195404, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5163297057151794, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7040897011756897, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.45445960760116577, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5819030404090881, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6447226405143738, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6595242023468018, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5938301086425781, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7438283562660217, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8469328880310059, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6745330691337585, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.1081027165055275, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1437283158302307, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.573423385620117, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.31600573658943176, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18556590378284454, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9355757832527161, "metrics/particle/class_1/pid_accuracy": 0.9776880145072937, "metrics/particle/class_2/efficiency": 0.5879515409469604, "metrics/particle/class_2/pid_accuracy": 0.8187369108200073, "metrics/particle/class_3/efficiency": 0.8386287093162537, "metrics/particle/class_3/pid_accuracy": 0.983193039894104, "metrics/particle/class_4/efficiency": 0.8017324209213257, "metrics/particle/class_4/pid_accuracy": 0.6302521228790283, "metrics/particle/class_5/efficiency": 0.9739999771118164, "metrics/particle/class_5/pid_accuracy": 0.673511266708374, "metrics/particle/count/bias_mean": -0.6712239384651184, "metrics/particle/count/mae": 2.861328125, "metrics/particle/count/prediction_mean": 69.384765625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05329938605427742, "metrics/particle/event/energy_response_mean": 0.9917013645172119, "metrics/particle/event/met_abs_error": 5.263102054595947, "metrics/particle/event/scalar_pt_relative_abs_error": 0.042945776134729385, "metrics/particle/event/scalar_pt_response_mean": 0.9989478588104248, "metrics/particle/event/vector_pt_closure": 0.04381285980343819, "metrics/particle/matched/delta_eta_abs_mean": 0.011965867131948471, "metrics/particle/matched/delta_phi_abs_mean": 0.01228686235845089, "metrics/particle/matched/delta_r_mean": 0.01928672380745411, "metrics/particle/matched/energy_relative_abs_mean": 0.11901843547821045, "metrics/particle/matched/pid_accuracy": 0.9663316607475281, "metrics/particle/matched/pt_relative_abs_mean": 0.11851955205202103, "metrics/particle/matching/duplicate_fraction": 0.04039409011602402, "metrics/particle/matching/efficiency": 0.8636693358421326, "metrics/particle/matching/f1": 0.8678267598152161, "metrics/particle/matching/purity": 0.872024416923523, "metrics/particle/matching/target_coverage_dr0p05": 0.8273237347602844, "metrics/particle/matching/target_coverage_dr0p10": 0.8940858244895935, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7136498093605042, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8895972371101379, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01955309510231018, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11818309128284454, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11716295033693314, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.002340339357033372, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15638884902000427, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005012264009565115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5522246956825256, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04800504446029663, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5703660249710083, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9377.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17920824885368347, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6868192553520203, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6773098111152649, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5336897373199463, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08862671256065369, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.82427453994751, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011231712996959686, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7909727692604065, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.024219276383519173, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.45684814453125, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10595.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8574433326721191, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.16960430145263672, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.900119781494141, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0290350914001465, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.16230808198451996, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.08956669270992279, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.962560176849365, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.559700846672058, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.007582664489746, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02037111110985279, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12440372258424759, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12355435639619827, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01178171206265688, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1610974669456482, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024635037407279015, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2167621850967407, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11287014186382294, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1769039630889893, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2192.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3450431525707245, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9258288145065308, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.8234024047851562, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.6768535375595093, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1391982138156891, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.910247802734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05345211550593376, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.1835453510284424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.028994979336857796, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.485423564910889, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2694.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7255875468254089, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.98832106590271, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.477299213409424, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0450994968414307, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.009224077686667442, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.004535771906375885, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.336539268493652, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.845695972442627, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.782880783081055, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01884857565164566, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11741995066404343, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11714305728673935, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008525402285158634, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15626123547554016, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000421874166931957, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.18863113224506378, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.025233536958694458, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5951864719390869, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18963.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13935790956020355, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8050090074539185, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6050270795822144, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46850085258483887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08070540428161621, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.020450592041016, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026003499515354633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3092205226421356, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.03096962720155716, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3609799146652222, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21151.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8845920562744141, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.629578709602356, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.210816383361816, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.012620449066162, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.22715766727924347, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7231956720352173, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.310276508331299, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5309289693832397, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.98440408706665, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019174257293343544, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11815571784973145, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11760945618152618, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.006859624292701483, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1566525399684906, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0035700248554348946, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.37411126494407654, "metrics/particle/momentum/all/accepted/pz_bias": -0.03851887211203575, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3046010732650757, "metrics/particle/momentum/all/accepted/samples": 30532.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16770663857460022, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7781004309654236, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7225688099861145, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5830762982368469, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.08771777153015137, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.353798866271973, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009233449585735798, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6040403246879578, "metrics/particle/momentum/all/assigned/pz_bias": -0.028738509863615036, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7161433696746826, "metrics/particle/momentum/all/assigned/samples": 34440.0, "metrics/particle/momentum/all/efficiency": 0.8626320958137512, "metrics/particle/momentum/all/event/energy_bias": -2.189138412475586, "metrics/particle/momentum/all/event/p3_error_mean": 16.276653289794922, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.49092435836792, "metrics/particle/momentum/all/event/px_bias": 0.39868980646133423, "metrics/particle/momentum/all/event/py_bias": -0.638164758682251, "metrics/particle/momentum/all/event/pz_abs_mean": 12.43038272857666, "metrics/particle/momentum/all/event/pz_bias": -2.936326026916504, "metrics/particle/momentum/all/event/pz_residual_rms": 22.266700744628906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0201638862490654, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09140469878911972, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09077376872301102, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00911039113998413, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12446939200162888, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0030024275183677673, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.36705705523490906, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05244436860084534, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.235360860824585, "metrics/particle/momentum/class_1/accepted/samples": 15654.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08153305947780609, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21121367812156677, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21786658465862274, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.116084523499012, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03440755978226662, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.3108115196228027, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0054519022814929485, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4715240001678467, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02985009364783764, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2374331951141357, "metrics/particle/momentum/class_1/assigned/samples": 16508.0, "metrics/particle/momentum/class_1/efficiency": 0.9380955100059509, "metrics/particle/momentum/class_1/event/energy_bias": 2.9833297729492188, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.564379692077637, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.442215919494629, "metrics/particle/momentum/class_1/event/px_bias": 0.06478066742420197, "metrics/particle/momentum/class_1/event/py_bias": -0.6234835386276245, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.734034538269043, "metrics/particle/momentum/class_1/event/pz_bias": -2.408029317855835, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.12922477722168, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02632615529000759, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16728471219539642, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.16700853407382965, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.003663330338895321, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.20612730085849762, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020785219967365265, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2826939821243286, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11782734841108322, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.087352752685547, "metrics/particle/momentum/class_2/accepted/samples": 1732.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5460195541381836, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5602039694786072, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7157122492790222, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4016569256782532, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2455252856016159, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9071598052978516, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04474708065390587, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1363675594329834, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0006741408724337816, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.371173858642578, "metrics/particle/momentum/class_2/assigned/samples": 2570.0, "metrics/particle/momentum/class_2/efficiency": 0.5817937254905701, "metrics/particle/momentum/class_2/event/energy_bias": -6.132012367248535, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.51047134399414, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.958982944488525, "metrics/particle/momentum/class_2/event/px_bias": 0.4614894390106201, "metrics/particle/momentum/class_2/event/py_bias": -0.27593153715133667, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.442526817321777, "metrics/particle/momentum/class_2/event/pz_bias": -0.2858138084411621, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.633808135986328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016960471868515015, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14375877380371094, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14337438344955444, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0032429329585283995, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18173955380916595, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016536734765395522, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23710018396377563, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00844133086502552, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.858519434928894, "metrics/particle/momentum/class_3/accepted/samples": 12699.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.19672368466854095, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.411458969116211, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2591732740402222, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.106667160987854, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.11922403424978256, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.724994659423828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007005253806710243, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4672880172729492, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.02692291885614395, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.49599289894104, "metrics/particle/momentum/class_3/assigned/samples": 14846.0, "metrics/particle/momentum/class_3/efficiency": 0.8350759744644165, "metrics/particle/momentum/class_3/event/energy_bias": 1.6580942869186401, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.248614311218262, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.242485046386719, "metrics/particle/momentum/class_3/event/px_bias": 0.02975306659936905, "metrics/particle/momentum/class_3/event/py_bias": 0.0837482362985611, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.083045482635498, "metrics/particle/momentum/class_3/event/pz_bias": -0.3867642283439636, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.828531265258789, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02277645468711853, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14246828854084015, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1389288306236267, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0522330105304718, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1817387491464615, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006825938355177641, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6976540088653564, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10033651441335678, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6954401731491089, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2813843786716461, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.499588966369629, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.050187587738037, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9530259370803833, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.13687150180339813, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.270357131958008, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013966480270028114, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9465013146400452, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.10816972702741623, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0331761837005615, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.8049450516700745, "metrics/particle/momentum/class_4/event/energy_bias": -0.7635507583618164, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2814230918884277, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5714927911758423, "metrics/particle/momentum/class_4/event/px_bias": 0.008464165031909943, "metrics/particle/momentum/class_4/event/py_bias": 0.08035561442375183, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3617585897445679, "metrics/particle/momentum/class_4/event/pz_bias": 0.12251334637403488, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.726880073547363, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013840438798069954, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1273265928030014, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12469568103551865, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02592751756310463, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16172656416893005, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019480518996715546, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5550719499588013, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.47544971108436584, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.9761550426483154, "metrics/particle/momentum/class_5/accepted/samples": 154.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033562738448381424, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.13902656733989716, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.13731306791305542, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02407296560704708, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.018987340852618217, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.18958351016044617, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5984997749328613, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.38163942098617554, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.0922741889953613, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9685534834861755, "metrics/particle/momentum/class_5/event/energy_bias": 0.06500076502561569, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6835659742355347, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2844488620758057, "metrics/particle/momentum/class_5/event/px_bias": -0.16579747200012207, "metrics/particle/momentum/class_5/event/py_bias": 0.09714645892381668, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8640049695968628, "metrics/particle/momentum/class_5/event/pz_bias": 0.021767865866422653, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9110381603240967, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2926805019378662, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2937159538269043, "metrics/set_slot/angular_recall_all_dr0p10": 0.9667955040931702, "metrics/set_slot/angular_recall_selected_dr0p10": 0.878594160079956, "metrics/set_slot/soft_count_bias_mean": 0.37112173438072205}, "step_train_time": 5.618452787399292, "step_valid_time": 54646.60474920273, "step_total_time": 54652.223202466965, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9976107334465485, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09978605354290493, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2915, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2899, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9629996696399075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9629996696399075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9629996696399075, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0370003303600925, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9577139081598943, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9577139081598943, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9577139081598943, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9629996696399075, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954299102587609, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09928182612174519, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2164, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2156, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566755083996463, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579459938025675, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573103295731032, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04205400619743249, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9531388152077808, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9544046038069942, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537712895377128, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566755083996463, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954961042354044, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10065227214605466, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1213, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9417701863354038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9454403741231489, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9436017113963439, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05455962587685113, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9423226812159002, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.94049008168028, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9417701863354038} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_48000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..bd84871beb94fca921e0207c38ea6905b493e06d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.274019329164298, "train": {"Classification_binary": 0.14402911067008972, "Cardinality": 0.08986274152994156, "Classification": 0.06408049911260605, "Regression_pt": 0.0813731923699379, "Regression_eta": 0.0010404582135379314, "Regression_sin_phi": 0.0005868945736438036, "Regression_cos_phi": 0.0005747622926719487, "Regression_energy": 0.08109083771705627, "HitGrouping": 0.1426703780889511, "Auxiliary": 0.17693160474300385, "Total": 0.7936617136001587, "Total_unweighted": 0.3727757549495436}, "train_group": {}, "valid": {"Classification_binary": 0.1226725975672404, "Cardinality": 0.08937795956929524, "Classification": 0.06103285153706869, "Regression_pt": 0.08376547694206238, "Regression_eta": 0.00106534780934453, "Regression_sin_phi": 0.0006292293546721339, "Regression_cos_phi": 0.0006284378857041398, "Regression_energy": 0.08324227730433147, "HitGrouping": 0.139320969581604, "Auxiliary": 0.16420961419741312, "Total": 0.764240582784017, "Total_unweighted": 0.35303621840042376, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8501161336898804, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.617023229598999, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2763766944408417, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6229273080825806, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6305642127990723, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3273826539516449, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8019925355911255, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5589281916618347, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2917655110359192, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5674712657928467, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5826046466827393, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4103693664073944, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8596820831298828, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.592041015625, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2572048306465149, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5957465767860413, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.6041163802146912, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30854251980781555, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.851429283618927, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5971100330352783, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2664741575717926, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6019569039344788, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6106976270675659, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32373860478401184, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026767157018184662, "metrics/hit_grouping/calorimeter/background_recall": 0.6523463726043701, "metrics/hit_grouping/calorimeter/completeness": 0.6815512776374817, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6934792399406433, "metrics/hit_grouping/calorimeter/energy_purity": 0.6856345534324646, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.2991165816783905, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8680769205093384, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5938869118690491, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21445207297801971, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6018035411834717, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5987948775291443, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2759629786014557, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8546217679977417, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.621739387512207, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2703784704208374, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6270936131477356, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.640781819820404, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.323871910572052, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8128650188446045, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5782660245895386, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37251928448677063, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5760868787765503, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.6045411825180054, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4247250556945801, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8430424928665161, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.656761646270752, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31205230951309204, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6640179753303528, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6869422793388367, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.35988789796829224, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8363330364227295, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7212408185005188, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4008372724056244, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7175070643424988, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7304592728614807, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3422291874885559, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8623774647712708, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5170965194702148, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20028123259544373, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5204511880874634, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5134046673774719, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2825097143650055, "metrics/hit_grouping/calorimeter/purity": 0.672187864780426, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7912992238998413, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.13327284157276154, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11230769008398056, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1321813315153122, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.09975188970565796, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03076923079788685, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7949439883232117, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2610892653465271, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1077537089586258, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2595154345035553, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.25098878145217896, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.25370582938194275, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8559597134590149, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6263228058815002, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2788890302181244, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6316656470298767, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6422770023345947, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3340110182762146, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7602320313453674, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38538554310798645, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5657957196235657, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6119764447212219, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5385534763336182, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8087403774261475, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2905215919017792, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5889034867286682, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6127013564109802, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41446998715400696, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.7008898854255676, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4137197732925415, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.48991909623146057, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5409834384918213, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6255808472633362, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.728729248046875, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3943888247013092, "metrics/hit_grouping/tracker/all/object_completeness": 0.5222379565238953, "metrics/hit_grouping/tracker/all/object_purity": 0.5694624781608582, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5802947282791138, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004755122121423483, "metrics/hit_grouping/tracker/background_recall": 0.04685186222195625, "metrics/hit_grouping/tracker/completeness": 0.5364885926246643, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.41144055128097534, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7664730548858643, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37931662797927856, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.59462571144104, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6208683252334595, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5294761061668396, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7066004276275635, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4018656015396118, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.49258488416671753, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5637843012809753, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6183133721351624, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.668455183506012, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4199029207229614, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3930536210536957, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4613814949989319, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6527103781700134, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7274710536003113, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.3964773416519165, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.521478533744812, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.588675856590271, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.575395405292511, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7031906843185425, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.3988015949726105, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4885071814060211, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5722002983093262, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6158455610275269, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.7348589301109314, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39089205861091614, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5291585326194763, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5439510345458984, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5804261565208435, "metrics/hit_grouping/tracker/purity": 0.511293351650238, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8935993313789368, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.059947870671749115, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.16512522101402283, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.16344279050827026, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07949609309434891, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7090545892715454, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2835666239261627, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.41356515884399414, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.43240538239479065, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5147745013237, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.708566427230835, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4524526298046112, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5831378102302551, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.640239953994751, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6558054685592651, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5916786193847656, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7388696670532227, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8415458798408508, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6794599294662476, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.10595129430294037, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.14865517616271973, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.630964279174805, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.313517302274704, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18307745456695557, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9358131885528564, "metrics/particle/class_1/pid_accuracy": 0.9785605669021606, "metrics/particle/class_2/efficiency": 0.590419590473175, "metrics/particle/class_2/pid_accuracy": 0.8153144717216492, "metrics/particle/class_3/efficiency": 0.8350889086723328, "metrics/particle/class_3/pid_accuracy": 0.9823511242866516, "metrics/particle/class_4/efficiency": 0.7911453247070312, "metrics/particle/class_4/pid_accuracy": 0.6459854245185852, "metrics/particle/class_5/efficiency": 0.9599999785423279, "metrics/particle/class_5/pid_accuracy": 0.6791666746139526, "metrics/particle/count/bias_mean": -0.9654948115348816, "metrics/particle/count/mae": 2.8444011211395264, "metrics/particle/count/prediction_mean": 69.09049224853516, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.054195087403059006, "metrics/particle/event/energy_response_mean": 0.9824156165122986, "metrics/particle/event/met_abs_error": 5.064348220825195, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04199844226241112, "metrics/particle/event/scalar_pt_response_mean": 0.9920980930328369, "metrics/particle/event/vector_pt_closure": 0.043689507991075516, "metrics/particle/matched/delta_eta_abs_mean": 0.012087555602192879, "metrics/particle/matched/delta_phi_abs_mean": 0.012324619106948376, "metrics/particle/matched/delta_r_mean": 0.019423404708504677, "metrics/particle/matched/energy_relative_abs_mean": 0.11647264659404755, "metrics/particle/matched/pid_accuracy": 0.966396152973175, "metrics/particle/matched/pt_relative_abs_mean": 0.1169092059135437, "metrics/particle/matching/duplicate_fraction": 0.038153842091560364, "metrics/particle/matching/efficiency": 0.8622846603393555, "metrics/particle/matching/f1": 0.8682677745819092, "metrics/particle/matching/purity": 0.8743345141410828, "metrics/particle/matching/target_coverage_dr0p05": 0.8255301713943481, "metrics/particle/matching/target_coverage_dr0p10": 0.8936490416526794, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7163076400756836, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8880174160003662, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01932554319500923, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11551755666732788, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11593670397996902, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004686850123107433, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1553807109594345, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005124372895807028, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5582708120346069, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.031069008633494377, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5676681995391846, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9367.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1724521964788437, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8207305669784546, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8585532307624817, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7227543592453003, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08863507956266403, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.946987152099609, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010949593968689442, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7916147708892822, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03419829532504082, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4604275226593018, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10594.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8565288782119751, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.3433456420898438, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.616280555725098, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9451699256896973, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13832435011863708, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5845519304275513, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.7043328285217285, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.9136533737182617, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.533469200134277, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.021134784445166588, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12525685131549835, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12545117735862732, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007918060757219791, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.164701908826828, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026532478630542755, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2687968015670776, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1046813428401947, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3823211193084717, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2186.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3426226079463959, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3047873973846436, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.2084834575653076, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.0598742961883545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.14306893944740295, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 63.298004150390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05226093530654907, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.0424153804779053, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09415711462497711, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.4109673500061035, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2698.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7236014604568481, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.495244264602661, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.461053848266602, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0837544202804565, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.049506835639476776, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10839860886335373, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.336830139160156, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5439873337745667, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.09339141845703, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019123032689094543, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11584755778312683, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11634553968906403, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011005423963069916, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.15619714558124542, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005282899364829063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.18896903097629547, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009546843357384205, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.637889564037323, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18929.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1352721005678177, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8680698871612549, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.649491012096405, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5181782841682434, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08033136278390884, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.85814905166626, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028875740244984627, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30804163217544556, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.009250345639884472, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.360253095626831, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21125.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8830060362815857, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.16122184693813324, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.649261474609375, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.616234302520752, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2189888209104538, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9412069320678711, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.959566354751587, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5746129751205444, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.140204906463623, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019329536706209183, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1164209321141243, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11687290668487549, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008842341601848602, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15657298266887665, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003805524669587612, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3798932433128357, "metrics/particle/momentum/all/accepted/pz_bias": -0.022983040660619736, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3521497249603271, "metrics/particle/momentum/all/accepted/samples": 30482.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1629711389541626, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8877331614494324, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9144462943077087, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7803968191146851, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.08780544251203537, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 19.27247428894043, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009239619597792625, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5928518772125244, "metrics/particle/momentum/all/assigned/pz_bias": -0.023585615679621696, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.300934076309204, "metrics/particle/momentum/all/assigned/samples": 34417.0, "metrics/particle/momentum/all/efficiency": 0.8612194061279297, "metrics/particle/momentum/all/event/energy_bias": -3.9998116493225098, "metrics/particle/momentum/all/event/p3_error_mean": 14.713162422180176, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.964702606201172, "metrics/particle/momentum/all/event/px_bias": -0.40681999921798706, "metrics/particle/momentum/all/event/py_bias": 1.4173601865768433, "metrics/particle/momentum/all/event/pz_abs_mean": 11.132147789001465, "metrics/particle/momentum/all/event/pz_bias": -3.0322535037994385, "metrics/particle/momentum/all/event/pz_residual_rms": 17.992782592773438, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019928056746721268, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.08777759224176407, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.08864805102348328, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002171205822378397, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12186112254858017, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003265672130510211, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37388548254966736, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.025665773078799248, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2665057182312012, "metrics/particle/momentum/class_1/accepted/samples": 15617.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07658527791500092, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19896920025348663, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.20534643530845642, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09963387250900269, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03506965562701225, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7759153842926025, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005572380498051643, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.46797606348991394, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.015770506113767624, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7882126569747925, "metrics/particle/momentum/class_1/assigned/samples": 16510.0, "metrics/particle/momentum/class_1/efficiency": 0.9358782172203064, "metrics/particle/momentum/class_1/event/energy_bias": 0.11614177376031876, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.670076370239258, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.929994583129883, "metrics/particle/momentum/class_1/event/px_bias": -0.4403785169124603, "metrics/particle/momentum/class_1/event/py_bias": 0.9788733124732971, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.171236038208008, "metrics/particle/momentum/class_1/event/pz_bias": -1.3339920043945312, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.574892044067383, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02598602883517742, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16900718212127686, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.16973565518856049, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.005039020907133818, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21032686531543732, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022196926176548004, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.320305585861206, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.04655054956674576, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3047235012054443, "metrics/particle/momentum/class_2/accepted/samples": 1757.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5007413029670715, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5931820273399353, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8556265830993652, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5608561038970947, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.23203125596046448, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.321839332580566, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04453124850988388, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.111161708831787, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.007903617806732655, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.420238971710205, "metrics/particle/momentum/class_2/assigned/samples": 2560.0, "metrics/particle/momentum/class_2/efficiency": 0.5901914834976196, "metrics/particle/momentum/class_2/event/energy_bias": -6.956815719604492, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.142884254455566, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.865665912628174, "metrics/particle/momentum/class_2/event/px_bias": 0.2787145674228668, "metrics/particle/momentum/class_2/event/py_bias": -0.06901197880506516, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.105886459350586, "metrics/particle/momentum/class_2/event/pz_bias": -0.8770592212677002, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.123186111450195, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01766373962163925, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1440749168395996, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1440213918685913, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01709107868373394, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18331515789031982, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014998421538621187, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23784486949443817, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.012071914970874786, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8558672070503235, "metrics/particle/momentum/class_3/accepted/samples": 12668.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.19918568432331085, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7105573415756226, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7223714590072632, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5841622352600098, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12082799524068832, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 29.209901809692383, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006944912485778332, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45186784863471985, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03379109874367714, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.79965341091156, "metrics/particle/momentum/class_3/assigned/samples": 14831.0, "metrics/particle/momentum/class_3/efficiency": 0.8330374360084534, "metrics/particle/momentum/class_3/event/energy_bias": 3.449512004852295, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.914522171020508, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 3.9859368801116943, "metrics/particle/momentum/class_3/event/px_bias": -0.05823754519224167, "metrics/particle/momentum/class_3/event/py_bias": 0.22797131538391113, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.95165491104126, "metrics/particle/momentum/class_3/event/pz_bias": -0.6177544593811035, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.444931030273438, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022356797009706497, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.13170789182186127, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.12854526937007904, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.031037122011184692, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1705828756093979, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.006968641187995672, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6244781017303467, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1370532065629959, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6887946128845215, "metrics/particle/momentum/class_4/accepted/samples": 287.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2796851098537445, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.879300594329834, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8383152484893799, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.710288405418396, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.15041783452033997, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.3796966075897217, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008356546051800251, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8441275954246521, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.013549708761274815, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1548643112182617, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.7884615659713745, "metrics/particle/momentum/class_4/event/energy_bias": -0.5793144702911377, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.102163791656494, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4638437032699585, "metrics/particle/momentum/class_4/event/px_bias": -0.04139650613069534, "metrics/particle/momentum/class_4/event/py_bias": 0.04862727224826813, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2674270868301392, "metrics/particle/momentum/class_4/event/pz_bias": 0.15304610133171082, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.0612897872924805, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01404167152941227, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11786753684282303, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12106125056743622, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008846556767821312, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.15290693938732147, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03267974033951759, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4961894750595093, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6821212768554688, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0182149410247803, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05176176875829697, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4117269217967987, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.29539304971694946, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.18128596246242523, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.019108280539512634, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.096891164779663, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03821656107902527, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7110284566879272, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4177592098712921, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.071356773376465, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": -0.029335366562008858, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6530280113220215, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2740974426269531, "metrics/particle/momentum/class_5/event/px_bias": -0.14552193880081177, "metrics/particle/momentum/class_5/event/py_bias": 0.23090019822120667, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8218225240707397, "metrics/particle/momentum/class_5/event/pz_bias": -0.35649406909942627, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9106833934783936, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2820359468460083, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.28326845169067383, "metrics/set_slot/angular_recall_all_dr0p10": 0.9657825827598572, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8784918785095215, "metrics/set_slot/soft_count_bias_mean": 0.2207782119512558}, "step_train_time": 5.658106088638306, "step_valid_time": 57575.62078857422, "step_total_time": 57581.278895139694, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.991343925716702, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10406766015295443, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3012, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2900, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616782292699042, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9664674634794157, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.964066898493128, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03353253652058433, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9580442682523951, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9628154050464808, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9604239112435834, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616782292699042, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.993446262858604, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09790530411696308, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9597701149425287, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9640319715808171, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9618963225520603, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03596802841918295, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571175950486295, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9613676731793961, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9592379264510412, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9597701149425287, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9893859177830249, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0959754534480397, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1212, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9409937888198758, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.947615324472244, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442929489676666, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05238467552775606, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9386645962732919, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9452697419859265, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.941955590183093, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9409937888198758} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_49000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..95d737575d314c7fd599f8eed88692a1f56f7435 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.3213947318552206, "train": {"Classification_binary": 0.14556580781936646, "Cardinality": 0.10080759227275848, "Classification": 0.060456641018390656, "Regression_pt": 0.0833873450756073, "Regression_eta": 0.0010728455381467938, "Regression_sin_phi": 0.0006138643948361278, "Regression_cos_phi": 0.0005695107975043356, "Regression_energy": 0.08342724293470383, "HitGrouping": 0.14099159836769104, "Auxiliary": 0.18155045807361603, "Total": 0.8117566108703613, "Total_unweighted": 0.3750932575785555}, "train_group": {}, "valid": {"Classification_binary": 0.12378119428952535, "Cardinality": 0.09286542733510335, "Classification": 0.06275023519992828, "Regression_pt": 0.08387838800748189, "Regression_eta": 0.0010624049852291744, "Regression_sin_phi": 0.0006550451895842949, "Regression_cos_phi": 0.0006698342816283306, "Regression_energy": 0.08363404870033264, "HitGrouping": 0.13992156585057577, "Auxiliary": 0.16785955429077148, "Total": 0.7793106238047282, "Total_unweighted": 0.35643115065370995, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8470045924186707, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6084719896316528, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.27363747358322144, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6149842739105225, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6254752278327942, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3346557021141052, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8007784485816956, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5586017966270447, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.294815331697464, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5673331022262573, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5809986591339111, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41206371784210205, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8549301624298096, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5877297520637512, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2582377791404724, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5917389988899231, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5992665886878967, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3174258768558502, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8475225567817688, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5917916893959045, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2664741575717926, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5970300436019897, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6060618162155151, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3314589560031891, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.026989014819264412, "metrics/hit_grouping/calorimeter/background_recall": 0.6542094349861145, "metrics/hit_grouping/calorimeter/completeness": 0.6797566413879395, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6922175884246826, "metrics/hit_grouping/calorimeter/energy_purity": 0.6844315528869629, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30041271448135376, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8629778027534485, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5892936587333679, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21254105865955353, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5974376797676086, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5961312651634216, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28002387285232544, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8527039289474487, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6180338263511658, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2723192572593689, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6239336133003235, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6339740753173828, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33830663561820984, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8094092607498169, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5697327256202698, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3745417892932892, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5681313276290894, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.597994327545166, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43319427967071533, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8416860103607178, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6545110940933228, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31653690338134766, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.662421464920044, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6849381923675537, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36555591225624084, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8328109383583069, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7232494950294495, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.3987441062927246, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7200820446014404, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7330878376960754, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3579277992248535, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8556795716285706, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5075427293777466, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1957278698682785, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.510983943939209, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5050100684165955, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29141557216644287, "metrics/hit_grouping/calorimeter/purity": 0.6705707311630249, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7952325940132141, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.12583386898040771, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10461538285017014, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1212252676486969, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08524592220783234, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7673237323760986, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23859989643096924, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0946408212184906, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23917587101459503, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.24076364934444427, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.26168757677078247, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.853249728679657, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.622037410736084, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2798071801662445, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6277536749839783, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6381730437278748, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3416185677051544, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7570840716362, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38493722677230835, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5600107908248901, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.6074331998825073, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5390018224716187, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8108586668968201, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2983735203742981, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5952998995780945, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6154830455780029, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4071789085865021, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6991133689880371, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4151374399662018, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.48814114928245544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5389052629470825, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6280223727226257, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.726847231388092, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3957585394382477, "metrics/hit_grouping/tracker/all/object_completeness": 0.5198819637298584, "metrics/hit_grouping/tracker/all/object_purity": 0.5670143961906433, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.581286609172821, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004749021492898464, "metrics/hit_grouping/tracker/background_recall": 0.04483800381422043, "metrics/hit_grouping/tracker/completeness": 0.534092903137207, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4135254919528961, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7632624506950378, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3709339499473572, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5909345746040344, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6236682534217834, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5343963503837585, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7059273719787598, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4104321300983429, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4949258863925934, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5570590496063232, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6177422404289246, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6682379245758057, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4352750778198242, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.38867050409317017, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4518280625343323, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6466423869132996, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7272183895111084, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4047447741031647, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5226234793663025, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5831446051597595, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.572429895401001, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.704846978187561, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4174433946609497, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.49613848328590393, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5730080604553223, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6005326509475708, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.730232298374176, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3802388310432434, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.5204852819442749, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.54494708776474, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5894404053688049, "metrics/hit_grouping/tracker/purity": 0.5090070366859436, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8886927366256714, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.05516941845417023, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1635223776102066, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1594572812318802, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07558644562959671, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7103190422058105, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2820114195346832, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4118461012840271, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4445968568325043, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5277345776557922, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7067371010780334, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.45499086380004883, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5806067585945129, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6363338232040405, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6561005711555481, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.5843285322189331, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7427088618278503, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8461073637008667, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6685028672218323, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.09860119223594666, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.13769812881946564, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.70094871520996, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3151409924030304, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18470118939876556, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9345073103904724, "metrics/particle/class_1/pid_accuracy": 0.9787846803665161, "metrics/particle/class_2/efficiency": 0.5814449191093445, "metrics/particle/class_2/pid_accuracy": 0.817287266254425, "metrics/particle/class_3/efficiency": 0.8222599029541016, "metrics/particle/class_3/pid_accuracy": 0.9815278649330139, "metrics/particle/class_4/efficiency": 0.7930702567100525, "metrics/particle/class_4/pid_accuracy": 0.6492718458175659, "metrics/particle/class_5/efficiency": 0.9639999866485596, "metrics/particle/class_5/pid_accuracy": 0.682572603225708, "metrics/particle/count/bias_mean": -1.1953125, "metrics/particle/count/mae": 2.9440104961395264, "metrics/particle/count/prediction_mean": 68.86067962646484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05272548273205757, "metrics/particle/event/energy_response_mean": 1.0033010244369507, "metrics/particle/event/met_abs_error": 5.123306751251221, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04221400246024132, "metrics/particle/event/scalar_pt_response_mean": 1.0106486082077026, "metrics/particle/event/vector_pt_closure": 0.042376723140478134, "metrics/particle/matched/delta_eta_abs_mean": 0.011727705597877502, "metrics/particle/matched/delta_phi_abs_mean": 0.01236224826425314, "metrics/particle/matched/delta_r_mean": 0.01918034255504608, "metrics/particle/matched/energy_relative_abs_mean": 0.11729778349399567, "metrics/particle/matched/pid_accuracy": 0.9663324356079102, "metrics/particle/matched/pt_relative_abs_mean": 0.11779799312353134, "metrics/particle/matching/duplicate_fraction": 0.03849862888455391, "metrics/particle/matching/efficiency": 0.85540771484375, "metrics/particle/matching/f1": 0.8627680540084839, "metrics/particle/matching/purity": 0.870256245136261, "metrics/particle/matching/target_coverage_dr0p05": 0.8255952000617981, "metrics/particle/matching/target_coverage_dr0p10": 0.8903499841690063, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7071631550788879, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8821812868118286, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01908649131655693, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.11485787481069565, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.11500004678964615, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013742901384830475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15446877479553223, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004931389354169369, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5497567653656006, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02099429816007614, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5883088111877441, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9328.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17515067756175995, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8151757717132568, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7750933766365051, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6502149701118469, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08689485490322113, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.93857479095459, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011630105786025524, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8148512840270996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.019182253628969193, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7886829376220703, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10576.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8529626727104187, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5882012844085693, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.637688636779785, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.991211175918579, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2759801149368286, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.38728031516075134, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.651637077331543, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7604995965957642, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.993537902832031, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.020704612135887146, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12330471724271774, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12258143723011017, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014558416791260242, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16225597262382507, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019266055896878242, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1705583333969116, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08231797814369202, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1379165649414062, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2180.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.34770864248275757, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9479162096977234, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.6662510633468628, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.5273007154464722, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.13962963223457336, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.647805213928223, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.050740741193294525, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.045241355895996, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04281855747103691, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.486658573150635, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2700.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.7216153740882874, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.5748541355133057, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.231313705444336, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.020059585571289, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10244514048099518, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05317097157239914, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.110116958618164, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5397577285766602, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.167457580566406, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018912697210907936, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.11678186058998108, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.11767390370368958, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.028940508142113686, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1583472341299057, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00047989762970246375, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19011184573173523, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02301485277712345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6350600123405457, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18754.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14238889515399933, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8678373098373413, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6478802561759949, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5305120944976807, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08487295359373093, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.60653305053711, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002897174097597599, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3137669563293457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02910233847796917, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4208327531814575, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21055.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8748425841331482, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.87744402885437, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.9147047996521, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.755562782287598, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.2887111008167267, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.8363587856292725, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.256108283996582, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9761016368865967, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.615674018859863, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019095351919531822, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.11665870249271393, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1172032281756401, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02321992628276348, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.157448872923851, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032053401228040457, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3715982735157013, "metrics/particle/momentum/all/accepted/pz_bias": -0.026664087548851967, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.31790030002594, "metrics/particle/momentum/all/accepted/samples": 30262.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16862909495830536, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8579123616218567, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7671604156494141, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.645781397819519, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.08980222046375275, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.039722442626953, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009350149892270565, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6043047308921814, "metrics/particle/momentum/all/assigned/pz_bias": -0.02712508663535118, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.449758768081665, "metrics/particle/momentum/all/assigned/samples": 34331.0, "metrics/particle/momentum/all/efficiency": 0.8550036549568176, "metrics/particle/momentum/all/event/energy_bias": 1.8907911777496338, "metrics/particle/momentum/all/event/p3_error_mean": 15.396560668945312, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.188161373138428, "metrics/particle/momentum/all/event/px_bias": 0.4622461199760437, "metrics/particle/momentum/all/event/py_bias": 1.2768099308013916, "metrics/particle/momentum/all/event/pz_abs_mean": 11.708742141723633, "metrics/particle/momentum/all/event/pz_bias": -3.2763590812683105, "metrics/particle/momentum/all/event/pz_residual_rms": 20.14636993408203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0200666431337595, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.08803455531597137, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.08907341957092285, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01634693704545498, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.12287283688783646, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00275924033485353, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3626978397369385, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.023931365460157394, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2209903001785278, "metrics/particle/momentum/class_1/accepted/samples": 15584.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07656006515026093, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18860575556755066, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1890541911125183, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09737427532672882, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.034729812294244766, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7585855722427368, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00582877965644002, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4660834074020386, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.018295127898454666, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7483989000320435, "metrics/particle/momentum/class_1/assigned/samples": 16470.0, "metrics/particle/momentum/class_1/efficiency": 0.9339006543159485, "metrics/particle/momentum/class_1/event/energy_bias": 5.309100151062012, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.661534309387207, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.516633033752441, "metrics/particle/momentum/class_1/event/px_bias": -0.08078929781913757, "metrics/particle/momentum/class_1/event/py_bias": 1.0845835208892822, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.796319961547852, "metrics/particle/momentum/class_1/event/pz_bias": -1.6818459033966064, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.11674690246582, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02639145404100418, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1690235584974289, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.16926231980323792, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02204485982656479, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.20885023474693298, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020278099924325943, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3044061660766602, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14278849959373474, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.192856788635254, "metrics/particle/momentum/class_2/accepted/samples": 1726.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5216696858406067, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.669891893863678, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.765255331993103, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4881187677383423, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.23299050331115723, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.4019200801849365, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04865506291389465, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.28832745552063, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.03373207151889801, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.066410541534424, "metrics/particle/momentum/class_2/assigned/samples": 2528.0, "metrics/particle/momentum/class_2/efficiency": 0.5797783136367798, "metrics/particle/momentum/class_2/event/energy_bias": -5.674795150756836, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.04873275756836, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.333893299102783, "metrics/particle/momentum/class_2/event/px_bias": 0.6618356108665466, "metrics/particle/momentum/class_2/event/py_bias": -0.19692352414131165, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.889253616333008, "metrics/particle/momentum/class_2/event/pz_bias": -1.421044111251831, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.96088218688965, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016886886209249496, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14432410895824432, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14443892240524292, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030735570937395096, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18437132239341736, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001039417926222086, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23432444036006927, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.016609670594334602, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8823078870773315, "metrics/particle/momentum/class_3/accepted/samples": 12507.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20913748443126678, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6075910329818726, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3894098997116089, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2608368396759033, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12586893141269684, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.376636505126953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006141594145447016, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45072680711746216, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.030288103967905045, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.9401254653930664, "metrics/particle/momentum/class_3/assigned/samples": 14817.0, "metrics/particle/momentum/class_3/efficiency": 0.8224501609802246, "metrics/particle/momentum/class_3/event/energy_bias": 2.4061243534088135, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.768060684204102, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 3.8231682777404785, "metrics/particle/momentum/class_3/event/px_bias": -0.03782058507204056, "metrics/particle/momentum/class_3/event/py_bias": 0.10964382439851761, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.839807510375977, "metrics/particle/momentum/class_3/event/pz_bias": -0.30215004086494446, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.837016105651855, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021773161366581917, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14461497962474823, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14002299308776855, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05991462245583534, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.189442440867424, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010273972526192665, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6934910416603088, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17501318454742432, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8385273218154907, "metrics/particle/momentum/class_4/accepted/samples": 292.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2823266088962555, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.265174388885498, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8986961841583252, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7981865406036377, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.14525139331817627, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.371597290039062, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01955307275056839, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.052196979522705, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.13915474712848663, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.9461567401885986, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.8021978139877319, "metrics/particle/momentum/class_4/event/energy_bias": -0.24748022854328156, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.382319211959839, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5965461730957031, "metrics/particle/momentum/class_4/event/px_bias": -0.03107414022088051, "metrics/particle/momentum/class_4/event/py_bias": 0.036650314927101135, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5122008323669434, "metrics/particle/momentum/class_4/event/pz_bias": 0.14906325936317444, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.954283237457275, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013276683166623116, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12660719454288483, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12518389523029327, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.052133817225694656, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16176718473434448, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3622382879257202, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.20180347561836243, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.568387031555176, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06086590886116028, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1427430808544159, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.14226149022579193, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04448603466153145, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03164556995034218, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.19932876527309418, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4556533098220825, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29139086604118347, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 2.871803045272827, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.9622641801834106, "metrics/particle/momentum/class_5/event/energy_bias": 0.09784159809350967, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6016649007797241, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.231431007385254, "metrics/particle/momentum/class_5/event/px_bias": -0.04990551248192787, "metrics/particle/momentum/class_5/event/py_bias": 0.24285580217838287, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8080760836601257, "metrics/particle/momentum/class_5/event/pz_bias": -0.020382286980748177, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9604458808898926, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2836732566356659, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2849421203136444, "metrics/set_slot/angular_recall_all_dr0p10": 0.9658383131027222, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8752764463424683, "metrics/set_slot/soft_count_bias_mean": -0.3953333795070648}, "step_train_time": 5.662152290344238, "step_valid_time": 60528.30474948883, "step_total_time": 60533.96690225601, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0127088215021012, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10428339584192847, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2918, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2904, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.96399074991741, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9646280991735537, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643093192333113, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035371900826446284, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9593657086223984, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.96, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9596827495042961, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.96399074991741, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0100179901722892, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1021174649533576, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2162, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2156, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9557913351016799, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9596094096759876, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576965669988926, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040390590324012425, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9531388152077808, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9569462938304483, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9550387596899225, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9557913351016799, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071203698444795, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09487980643001026, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1213, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9417701863354038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9432348367029549, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9425019425019425, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056765163297045104, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9394409937888198, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9409020217729394, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9401709401709402, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9417701863354038} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_5000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..2493888b8624256e2c5513351fb2170821476fec --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23687701345461437, "train": {"Classification_binary": 0.32814842462539673, "Cardinality": 0.2722207009792328, "Classification": 0.22310194373130798, "Regression_pt": 0.35237908363342285, "Regression_eta": 0.004441272467374802, "Regression_sin_phi": 0.00226184562779963, "Regression_cos_phi": 0.002345035318285227, "Regression_energy": 0.351691335439682, "HitGrouping": 0.22986197471618652, "Auxiliary": 0.4234206974506378, "Total": 2.228592872619629, "Total_unweighted": 1.2643689408432692}, "train_group": {}, "valid": {"Classification_binary": 0.324428915977478, "Cardinality": 0.26383642355600995, "Classification": 0.2284640073776245, "Regression_pt": 0.349958340326945, "Regression_eta": 0.004503834061324596, "Regression_sin_phi": 0.0024227223669489226, "Regression_cos_phi": 0.0023918570950627327, "Regression_energy": 0.34730227788289386, "HitGrouping": 0.23138153553009033, "Auxiliary": 0.41741812229156494, "Total": 2.2287143071492515, "Total_unweighted": 1.2594719550882776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8117351531982422, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2997431457042694, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37206006050109863, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.29858705401420593, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2741541564464569, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.44620761275291443, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7657116055488586, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.32743826508522034, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.41816332936286926, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.32623305916786194, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.27303677797317505, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5062690377235413, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8250734210014343, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.28883063793182373, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3389629125595093, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.288185715675354, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2674635648727417, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.42356160283088684, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.815456748008728, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2958051860332489, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3567173182964325, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.29494544863700867, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2701164484024048, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.43826746940612793, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02525004930794239, "metrics/hit_grouping/calorimeter/background_recall": 0.6235926747322083, "metrics/hit_grouping/calorimeter/completeness": 0.40765586495399475, "metrics/hit_grouping/calorimeter/energy_completeness": 0.43359681963920593, "metrics/hit_grouping/calorimeter/energy_purity": 0.42839840054512024, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5337613821029663, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8438301682472229, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.2976457476615906, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2976410984992981, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2990623414516449, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.26584914326667786, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3730068802833557, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8010333776473999, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3140028119087219, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3753032386302948, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.31271612644195557, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.30044278502464294, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4741630256175995, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7704298496246338, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2729456424713135, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4623941481113434, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2677127420902252, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.24754607677459717, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5389963388442993, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.779119610786438, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.32367751002311707, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.43052008748054504, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.324387788772583, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3100971281528473, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5212706327438354, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7707822322845459, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.499661386013031, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.628466784954071, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4923818111419678, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4515857696533203, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5159602165222168, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8602381348609924, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.23975452780723572, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24260078370571136, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2380286157131195, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.20391330122947693, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3390920162200928, "metrics/hit_grouping/calorimeter/purity": 0.40142953395843506, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9070103764533997, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07419651001691818, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13230769336223602, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07333333790302277, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.017474206164479256, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03846153989434242, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8831384778022766, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.058618344366550446, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09007981419563293, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.057940635830163956, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.03668311610817909, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2234891653060913, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8096125721931458, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.31417056918144226, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37683629989624023, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3133004605770111, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.28892743587493896, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.45914217829704285, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6662878394126892, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.454423189163208, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.32223519682884216, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.30729779601097107, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7062163949012756, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7523671984672546, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.45989903807640076, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4079859256744385, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3251483738422394, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5277621746063232, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5982024669647217, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4292352497577667, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2583670914173126, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.26125288009643555, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7653776407241821, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6327056884765625, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4397789537906647, "metrics/hit_grouping/tracker/all/object_completeness": 0.29115450382232666, "metrics/hit_grouping/tracker/all/object_purity": 0.28118759393692017, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7266672849655151, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013054471462965012, "metrics/hit_grouping/tracker/background_recall": 0.06669430434703827, "metrics/hit_grouping/tracker/completeness": 0.31324803829193115, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5896519422531128, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6402613520622253, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4479271173477173, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.34973326325416565, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.34289810061454773, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7302049994468689, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6152153611183167, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42965924739837646, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.24946410953998566, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.24902385473251343, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7487150430679321, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6345168948173523, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43244338035583496, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2054138332605362, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.17837272584438324, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6953883767127991, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6456568241119385, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4612688720226288, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2880008816719055, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2763292193412781, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7180086374282837, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6757981777191162, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5186418294906616, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3520466685295105, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3472755253314972, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.661784291267395, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6082565188407898, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39791616797447205, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2845557928085327, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2758961021900177, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7493561506271362, "metrics/hit_grouping/tracker/purity": 0.30104735493659973, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8761642575263977, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01824500411748886, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.028090856969356537, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.028365246951580048, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08123371005058289, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6628416776657104, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.16640746593475342, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.11405380815267563, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.09777583926916122, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5837221145629883, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5961923003196716, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5281860828399658, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.347066193819046, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.336426317691803, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8306475281715393, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4796040654182434, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.37341344356536865, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.38937753438949585, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5447291731834412, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.006123265251517296, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.013924438506364822, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.70052146911621, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.19887059926986694, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.06843077391386032, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5456668138504028, "metrics/particle/class_1/pid_accuracy": 0.9151497483253479, "metrics/particle/class_2/efficiency": 0.3582005798816681, "metrics/particle/class_2/pid_accuracy": 0.3341684937477112, "metrics/particle/class_3/efficiency": 0.4780855178833008, "metrics/particle/class_3/pid_accuracy": 0.9102984070777893, "metrics/particle/class_4/efficiency": 0.48508182168006897, "metrics/particle/class_4/pid_accuracy": 0.2599206268787384, "metrics/particle/class_5/efficiency": 0.5519999861717224, "metrics/particle/class_5/pid_accuracy": 0.25724637508392334, "metrics/particle/count/bias_mean": -12.334635734558105, "metrics/particle/count/mae": 12.611979484558105, "metrics/particle/count/prediction_mean": 57.72135543823242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.12498193234205246, "metrics/particle/event/energy_response_mean": 0.984014093875885, "metrics/particle/event/met_abs_error": 13.620437622070312, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10808641463518143, "metrics/particle/event/scalar_pt_response_mean": 1.0064774751663208, "metrics/particle/event/vector_pt_closure": 0.10545516014099121, "metrics/particle/matched/delta_eta_abs_mean": 0.02060888335108757, "metrics/particle/matched/delta_phi_abs_mean": 0.020288895815610886, "metrics/particle/matched/delta_r_mean": 0.03229924663901329, "metrics/particle/matched/energy_relative_abs_mean": 0.19577915966510773, "metrics/particle/matched/pid_accuracy": 0.8691767454147339, "metrics/particle/matched/pt_relative_abs_mean": 0.19264408946037292, "metrics/particle/matching/duplicate_fraction": 0.1042296439409256, "metrics/particle/matching/efficiency": 0.5003066658973694, "metrics/particle/matching/f1": 0.5486024022102356, "metrics/particle/matching/purity": 0.6072185635566711, "metrics/particle/matching/target_coverage_dr0p05": 0.4471311867237091, "metrics/particle/matching/target_coverage_dr0p10": 0.5936192870140076, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.31327250599861145, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5618460178375244, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03140701353549957, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1972903162240982, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1933649480342865, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013952595181763172, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2348853349685669, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.015168539248406887, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.212007999420166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07010062783956528, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7309083938598633, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5340.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4490703344345093, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.0450775623321533, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.2447314262390137, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.059497594833374, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.28390705585479736, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.60757827758789, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03028828650712967, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8048866987228394, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.133943572640419, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.152082443237305, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8221.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4882955253124237, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.13674306869506836, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 14.169659614562988, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.348621845245361, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.9583840370178223, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5690367817878723, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.12930679321289, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.23820587992668152, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.472082138061523, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03076862171292305, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20009742677211761, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19589856266975403, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.006043667439371347, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23872658610343933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.037779491394758224, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.02958607673645, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.009707137942314148, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.239985466003418, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1297.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5133515000343323, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1480634212493896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.107231616973877, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.9061827659606934, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2639034688472748, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.59562110900879, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0645330548286438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9431917667388916, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.08817703276872635, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.2045745849609375, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1906.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4293280243873596, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -10.321744918823242, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.703543663024902, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.990030288696289, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.043558474630117416, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05569801479578018, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 14.494124412536621, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7509105205535889, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 32.7451171875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.032843198627233505, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19333294034004211, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1908557116985321, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012013573199510574, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.232176274061203, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001259785844013095, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.42457687854766846, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.027968982234597206, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0477136373519897, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11113.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5308858752250671, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.232539653778076, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.626976728439331, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.448052406311035, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3378655016422272, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 15.335447311401367, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011391674168407917, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9668710827827454, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.025997519493103027, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.290886640548706, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19049.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5184027552604675, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 8.93875503540039, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.834508895874023, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.310835838317871, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.3411524295806885, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.1940340995788574, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.301558494567871, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4738953113555908, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.321372985839844, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.032259535044431686, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19501778483390808, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1919790655374527, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01127746794372797, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23347778618335724, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008112676441669464, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7787503600120544, "metrics/particle/momentum/all/accepted/pz_bias": 0.03930967301130295, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.060227394104004, "metrics/particle/momentum/all/accepted/samples": 17750.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5066870450973511, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.7617709636688232, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.5506443977355957, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.3684966564178467, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3178297281265259, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.30915069580078, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.020187824964523315, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.3321094512939453, "metrics/particle/momentum/all/assigned/pz_bias": 0.06047580763697624, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.9140074253082275, "metrics/particle/momentum/all/assigned/samples": 29176.0, "metrics/particle/momentum/all/efficiency": 0.5014974474906921, "metrics/particle/momentum/all/event/energy_bias": -1.246247410774231, "metrics/particle/momentum/all/event/p3_error_mean": 34.254066467285156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.39358901977539, "metrics/particle/momentum/all/event/px_bias": 4.255978107452393, "metrics/particle/momentum/all/event/py_bias": 2.7073726654052734, "metrics/particle/momentum/all/event/pz_abs_mean": 24.97307777404785, "metrics/particle/momentum/all/event/pz_bias": -1.5152205228805542, "metrics/particle/momentum/all/event/pz_residual_rms": 37.22217559814453, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.039924297481775284, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2120797336101532, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.20845843851566315, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0029304414056241512, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24848230183124542, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.009309967048466206, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.912340521812439, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03174338862299919, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.21577525138855, "metrics/particle/momentum/class_1/accepted/samples": 9130.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.38253888487815857, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.348708152770996, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.4050934314727783, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.2024316787719727, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2958853244781494, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.5901570320129395, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.016334688290953636, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.2587085962295532, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.053260378539562225, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.2261247634887695, "metrics/particle/momentum/class_1/assigned/samples": 14509.0, "metrics/particle/momentum/class_1/efficiency": 0.5471324920654297, "metrics/particle/momentum/class_1/event/energy_bias": 24.184520721435547, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.384124755859375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.42919158935547, "metrics/particle/momentum/class_1/event/px_bias": 5.806276321411133, "metrics/particle/momentum/class_1/event/py_bias": 1.468100905418396, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.524141311645508, "metrics/particle/momentum/class_1/event/pz_bias": -3.49230694770813, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.067405700683594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03813517466187477, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2265545278787613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21941949427127838, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03975827619433403, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2603698670864105, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03477443754673004, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9168967008590698, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08822616934776306, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.135177135467529, "metrics/particle/momentum/class_2/accepted/samples": 1064.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5271203517913818, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7362420558929443, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9317017197608948, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.600195586681366, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2925095558166504, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.173365116119385, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.051394205540418625, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.612525701522827, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.25714778900146484, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.603708744049072, "metrics/particle/momentum/class_2/assigned/samples": 1829.0, "metrics/particle/momentum/class_2/efficiency": 0.3574067950248718, "metrics/particle/momentum/class_2/event/energy_bias": -20.877901077270508, "metrics/particle/momentum/class_2/event/p3_error_mean": 24.90650749206543, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.10028076171875, "metrics/particle/momentum/class_2/event/px_bias": 1.01698899269104, "metrics/particle/momentum/class_2/event/py_bias": -1.1593159437179565, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.773395538330078, "metrics/particle/momentum/class_2/event/pz_bias": 1.5569491386413574, "metrics/particle/momentum/class_2/event/pz_residual_rms": 30.461156845092773, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021625198423862457, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16943703591823578, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1676526516675949, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03720473498106003, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20886671543121338, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021935838740319014, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4154019057750702, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02304980903863907, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1418782472610474, "metrics/particle/momentum/class_3/accepted/samples": 7294.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6505750417709351, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.724391937255859, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.151188373565674, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.015782356262207, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.34702423214912415, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.88580322265625, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01916727051138878, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1997700929641724, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.023779217153787613, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.07177734375, "metrics/particle/momentum/class_3/assigned/samples": 12417.0, "metrics/particle/momentum/class_3/efficiency": 0.47964751720428467, "metrics/particle/momentum/class_3/event/energy_bias": -1.2619082927703857, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.447858810424805, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.271452903747559, "metrics/particle/momentum/class_3/event/px_bias": -2.3973729610443115, "metrics/particle/momentum/class_3/event/py_bias": 1.215804100036621, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.598270416259766, "metrics/particle/momentum/class_3/event/pz_bias": -0.22526127099990845, "metrics/particle/momentum/class_3/event/pz_residual_rms": 22.276784896850586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.037324272096157074, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16113877296447754, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1603306233882904, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.021202150732278824, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20316405594348907, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.016853932291269302, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.5149728059768677, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.9086329340934753, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.286259889602661, "metrics/particle/momentum/class_4/accepted/samples": 178.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5809081792831421, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.12394118309021, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.421705961227417, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.257434129714966, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.35197368264198303, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.824458122253418, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02302631549537182, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.7796608209609985, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.6822664141654968, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.149719715118408, "metrics/particle/momentum/class_4/assigned/samples": 304.0, "metrics/particle/momentum/class_4/efficiency": 0.4890109896659851, "metrics/particle/momentum/class_4/event/energy_bias": -1.415729284286499, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.076834201812744, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.8605313301086426, "metrics/particle/momentum/class_4/event/px_bias": -0.043873123824596405, "metrics/particle/momentum/class_4/event/py_bias": 0.36194321513175964, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.504578113555908, "metrics/particle/momentum/class_4/event/pz_bias": 0.5667300224304199, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.372556686401367, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0374307744204998, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23413394391536713, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23265893757343292, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07038741558790207, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2713877260684967, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8329182863235474, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1881541907787323, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.2521376609802246, "metrics/particle/momentum/class_5/accepted/samples": 84.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11927177011966705, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4271945059299469, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.389022558927536, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07858576625585556, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.24786324799060822, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6238540410995483, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.1111111119389534, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.3004262447357178, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1597314178943634, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.596197128295898, "metrics/particle/momentum/class_5/assigned/samples": 117.0, "metrics/particle/momentum/class_5/efficiency": 0.5283018946647644, "metrics/particle/momentum/class_5/event/energy_bias": -1.8752306699752808, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.703723430633545, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.686971664428711, "metrics/particle/momentum/class_5/event/px_bias": -0.12604114413261414, "metrics/particle/momentum/class_5/event/py_bias": 0.8208403587341309, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.359023332595825, "metrics/particle/momentum/class_5/event/pz_bias": 0.07866832613945007, "metrics/particle/momentum/class_5/event/pz_residual_rms": 7.524059772491455, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6410221457481384, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6429638266563416, "metrics/set_slot/angular_recall_all_dr0p10": 0.8336338400840759, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6286173462867737, "metrics/set_slot/soft_count_bias_mean": -4.014482498168945}, "step_train_time": 5.243337154388428, "step_valid_time": 15113.203694581985, "step_total_time": 15118.447031974792, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0694736041578226, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27114222127323173, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2956, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2615, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2452, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8638916418896597, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8846414073071719, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8741434063179008, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11535859269282815, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8100429468120252, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8294993234100135, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8196556911248537, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8638916418896597, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0235300698658194, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25130517743140723, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1979, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1933, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8748894783377542, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8819073083778967, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8783843763870395, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11809269162210338, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8545534924845269, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8614081996434938, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8579671549045716, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8748894783377542, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016655522577705, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2566992680197109, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1235, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1078, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1035, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8369565217391305, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8728744939271255, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8545382481173206, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12712550607287448, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8035714285714286, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8380566801619433, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8204518430439952, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8369565217391305} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_50000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..55a974763c0fc51815ea85563389df944ba257bb --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3687701345461436, "train": {"Classification_binary": 0.15057599544525146, "Cardinality": 0.13159315288066864, "Classification": 0.056961189955472946, "Regression_pt": 0.08129727840423584, "Regression_eta": 0.0009582805796526372, "Regression_sin_phi": 0.000602935382630676, "Regression_cos_phi": 0.0006187756662257016, "Regression_energy": 0.08023669570684433, "HitGrouping": 0.1392974555492401, "Auxiliary": 0.1871606558561325, "Total": 0.8462792038917542, "Total_unweighted": 0.3712511511403136}, "train_group": {}, "valid": {"Classification_binary": 0.12604807813962302, "Cardinality": 0.09352029363314311, "Classification": 0.06310149530569713, "Regression_pt": 0.08262356122334798, "Regression_eta": 0.0011556509416550398, "Regression_sin_phi": 0.000650246317187945, "Regression_cos_phi": 0.0006571345341702303, "Regression_energy": 0.08220911026000977, "HitGrouping": 0.14117116729418436, "Auxiliary": 0.1696763038635254, "Total": 0.7854202588399252, "Total_unweighted": 0.35644527672169113, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8475381731987, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5982487797737122, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.27410975098609924, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6034480333328247, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6106373071670532, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33578917384147644, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8006488680839539, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5567741394042969, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.29041004180908203, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5645486116409302, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5785015821456909, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4079972803592682, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8529307842254639, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5804060101509094, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2640739679336548, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5846226811408997, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.585224449634552, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3180972933769226, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8465059995651245, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5840279459953308, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2696656584739685, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5888800024986267, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5927990674972534, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33185410499572754, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.02712000161409378, "metrics/hit_grouping/calorimeter/background_recall": 0.6553938388824463, "metrics/hit_grouping/calorimeter/completeness": 0.6741985082626343, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6873838901519775, "metrics/hit_grouping/calorimeter/energy_purity": 0.679675817489624, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30816736817359924, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8607928156852722, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5788671374320984, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21492983400821686, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5863903760910034, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5786946415901184, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2831293046474457, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8527402281761169, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6119081974029541, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.278990775346756, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.616468608379364, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.621622622013092, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33284813165664673, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8097688555717468, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5658981204032898, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3758058547973633, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5653998255729675, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5926170349121094, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4339527189731598, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8409991264343262, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6513436436653137, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3197134733200073, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6587327122688293, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6773520112037659, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36524447798728943, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8294905424118042, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.7177782654762268, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4076399803161621, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.7153719663619995, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.728818416595459, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3699633777141571, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8546035885810852, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4945443272590637, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19820544123649597, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4975975453853607, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48449385166168213, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2910807430744171, "metrics/hit_grouping/calorimeter/purity": 0.6651772856712341, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7886660695075989, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.12072108685970306, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11230769008398056, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11958607286214828, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07263128459453583, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.032307691872119904, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7686807513237, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.21674996614456177, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09122006595134735, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.21568724513053894, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2054809182882309, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2502850592136383, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8522149920463562, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6150272488594055, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2832830548286438, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6203470826148987, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6261630058288574, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34293022751808167, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7582868933677673, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3883741796016693, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5570194125175476, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5975085496902466, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5390018224716187, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.8086947798728943, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28435221314430237, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5957331657409668, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.6186085343360901, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40381380915641785, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.7034600377082825, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4160037934780121, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.491670697927475, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5385560393333435, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6197527050971985, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7296519875526428, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39618363976478577, "metrics/hit_grouping/tracker/all/object_completeness": 0.5210896134376526, "metrics/hit_grouping/tracker/all/object_purity": 0.5639312863349915, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5760438442230225, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004392158705741167, "metrics/hit_grouping/tracker/background_recall": 0.04371260851621628, "metrics/hit_grouping/tracker/completeness": 0.5335976481437683, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4177326261997223, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7666422128677368, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.37886103987693787, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5926286578178406, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.6150351166725159, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5259225368499756, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7066226601600647, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4111936092376709, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.49424421787261963, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.554387629032135, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6167904138565063, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6720075607299805, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4186893105506897, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.39080581068992615, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4606277346611023, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6440129280090332, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7293938994407654, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.3990834057331085, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5224693417549133, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5824986696243286, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.567307710647583, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7083712816238403, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4047936201095581, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.49581795930862427, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5804327726364136, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.606524646282196, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.733730137348175, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39089205861091614, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.523735761642456, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5368411540985107, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.5820651054382324, "metrics/hit_grouping/tracker/purity": 0.5083528161048889, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8887549042701721, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.056907039135694504, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.15975458920001984, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1596667766571045, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07688966393470764, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7154523134231567, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28149300813674927, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.410445898771286, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.43867847323417664, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5189217329025269, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.7096494436264038, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4553450345993042, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5827875137329102, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6331260800361633, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6503748297691345, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.6128445863723755, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7455012798309326, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8492230176925659, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6730560064315796, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.12711721658706665, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.1422511786222458, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.65504264831543, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30646035075187683, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1760205328464508, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9325088858604431, "metrics/particle/class_1/pid_accuracy": 0.9789513945579529, "metrics/particle/class_2/efficiency": 0.5757235884666443, "metrics/particle/class_2/pid_accuracy": 0.8183943629264832, "metrics/particle/class_3/efficiency": 0.8210585117340088, "metrics/particle/class_3/pid_accuracy": 0.9818143844604492, "metrics/particle/class_4/efficiency": 0.7940327525138855, "metrics/particle/class_4/pid_accuracy": 0.6254545450210571, "metrics/particle/class_5/efficiency": 0.9580000042915344, "metrics/particle/class_5/pid_accuracy": 0.668058454990387, "metrics/particle/count/bias_mean": -1.439453125, "metrics/particle/count/mae": 2.9915363788604736, "metrics/particle/count/prediction_mean": 68.61653900146484, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.05323508381843567, "metrics/particle/event/energy_response_mean": 0.9957166314125061, "metrics/particle/event/met_abs_error": 4.94622278213501, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04087330028414726, "metrics/particle/event/scalar_pt_response_mean": 1.0021252632141113, "metrics/particle/event/vector_pt_closure": 0.042461544275283813, "metrics/particle/matched/delta_eta_abs_mean": 0.012443832121789455, "metrics/particle/matched/delta_phi_abs_mean": 0.012467271648347378, "metrics/particle/matched/delta_r_mean": 0.019868293777108192, "metrics/particle/matched/energy_relative_abs_mean": 0.11961948126554489, "metrics/particle/matched/pid_accuracy": 0.9663752317428589, "metrics/particle/matched/pt_relative_abs_mean": 0.1199609786272049, "metrics/particle/matching/duplicate_fraction": 0.038531240075826645, "metrics/particle/matching/efficiency": 0.8534561395645142, "metrics/particle/matching/f1": 0.8623151779174805, "metrics/particle/matching/purity": 0.8713601231575012, "metrics/particle/matching/target_coverage_dr0p05": 0.8154842853546143, "metrics/particle/matching/target_coverage_dr0p10": 0.8861773610115051, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7023400068283081, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8803133368492126, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01969398371875286, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1185612678527832, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1189727932214737, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02598242089152336, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.15889526903629303, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004656199365854263, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5509629845619202, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.029559636488556862, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.5028499364852905, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9235.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1776130050420761, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7091752886772156, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6673156023025513, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5467061400413513, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09400492906570435, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.466254711151123, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010529311373829842, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8122126460075378, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.054016124457120895, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6722137928009033, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10542.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8444586396217346, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.26187586784362793, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.550019264221191, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9595093727111816, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2116219699382782, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21088111400604248, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.605393409729004, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.872320294380188, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.084555625915527, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.020810525864362717, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.12858863174915314, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.12731972336769104, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.029938172549009323, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.16645602881908417, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02398524060845375, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2833952903747559, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0045957681722939014, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5953497886657715, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 2168.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.3233279883861542, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.171395182609558, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.4596035480499268, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.3169636726379395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1393258422613144, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 62.562721252441406, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05018726736307144, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.089099168777466, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.026472410187125206, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 5.570179462432861, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2670.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.717643141746521, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.1431849002838135, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.345767974853516, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0803054571151733, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05913706496357918, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.062384121119976044, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.209355354309082, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.31450948119163513, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.45479965209961, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019577080383896828, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1182650700211525, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1190081313252449, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.026363976299762726, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1588103026151657, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00042723631486296654, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19309838116168976, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004733335692435503, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6478261947631836, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18725.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14315690100193024, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8463883399963379, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6703434586524963, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5489110946655273, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08400361984968185, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.618927001953125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002855647122487426, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3185066282749176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0002751474967226386, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.545098066329956, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21011.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8734897375106812, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3285598754882812, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.437836170196533, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.349290370941162, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1497417390346527, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2607359290122986, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9504058361053467, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1401292383670807, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.285650730133057, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019701670855283737, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1190987303853035, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.11959539353847504, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.026504216715693474, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.15939870476722717, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0034187466371804476, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3812503516674042, "metrics/particle/momentum/all/accepted/pz_bias": 0.006449653301388025, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.372345209121704, "metrics/particle/momentum/all/accepted/samples": 30128.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.16782724857330322, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8294776678085327, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8090047240257263, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6861712336540222, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09140051901340485, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 19.271148681640625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008912134915590286, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6087245941162109, "metrics/particle/momentum/all/assigned/pz_bias": 0.01853543519973755, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.4669721126556396, "metrics/particle/momentum/all/assigned/samples": 34223.0, "metrics/particle/momentum/all/efficiency": 0.8512177467346191, "metrics/particle/momentum/all/event/energy_bias": -0.5527490377426147, "metrics/particle/momentum/all/event/p3_error_mean": 14.568695068359375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.738357067108154, "metrics/particle/momentum/all/event/px_bias": 0.30222663283348083, "metrics/particle/momentum/all/event/py_bias": -0.11223899573087692, "metrics/particle/momentum/all/event/pz_abs_mean": 11.205622673034668, "metrics/particle/momentum/all/event/pz_bias": 0.697939932346344, "metrics/particle/momentum/all/event/pz_residual_rms": 18.577415466308594, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01985519751906395, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.09246945381164551, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.09380961954593658, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.029099371284246445, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1279682070016861, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0030227024108171463, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37271514534950256, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.022496569901704788, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.284745454788208, "metrics/particle/momentum/class_1/accepted/samples": 15549.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07773210108280182, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18101516366004944, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.17983072996139526, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09556300193071365, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03506959229707718, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.1541123390197754, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005652464460581541, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4707077741622925, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0177628006786108, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7923991680145264, "metrics/particle/momentum/class_1/assigned/samples": 16453.0, "metrics/particle/momentum/class_1/efficiency": 0.9318032264709473, "metrics/particle/momentum/class_1/event/energy_bias": 3.426466226577759, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.105268478393555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.216562271118164, "metrics/particle/momentum/class_1/event/px_bias": 0.185402512550354, "metrics/particle/momentum/class_1/event/py_bias": -0.13001203536987305, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.338827133178711, "metrics/particle/momentum/class_1/event/pz_bias": 0.7416343688964844, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.857099533081055, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.025848954916000366, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1731526106595993, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1735021471977234, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0007454921724274755, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2132115662097931, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02115158550441265, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3695992231369019, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.052999984472990036, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.439074993133545, "metrics/particle/momentum/class_2/accepted/samples": 1702.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5235450863838196, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6500827074050903, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6810795664787292, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.36589181423187256, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2260904312133789, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.443150520324707, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04361744597554207, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.231018304824829, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1237674355506897, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.6815361976623535, "metrics/particle/momentum/class_2/assigned/samples": 2499.0, "metrics/particle/momentum/class_2/efficiency": 0.5717164874076843, "metrics/particle/momentum/class_2/event/energy_bias": -7.830992221832275, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.42159652709961, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.93597936630249, "metrics/particle/momentum/class_2/event/px_bias": 0.12657412886619568, "metrics/particle/momentum/class_2/event/py_bias": -0.39453476667404175, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.489070892333984, "metrics/particle/momentum/class_2/event/pz_bias": -0.8047031164169312, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.67536163330078, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018696870654821396, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14442011713981628, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14391465485095978, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.025918522849678993, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18369002640247345, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011257638689130545, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23752136528491974, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.013393609784543514, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8444070219993591, "metrics/particle/momentum/class_3/accepted/samples": 12436.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2068573236465454, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5493651628494263, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.505322813987732, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3710612058639526, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13076193630695343, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 29.16533088684082, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006236442364752293, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.46860450506210327, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.006169595755636692, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.177457571029663, "metrics/particle/momentum/class_3/assigned/samples": 14752.0, "metrics/particle/momentum/class_3/efficiency": 0.8177812695503235, "metrics/particle/momentum/class_3/event/energy_bias": 4.2409563064575195, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.919797420501709, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.030593395233154, "metrics/particle/momentum/class_3/event/px_bias": -0.1578369140625, "metrics/particle/momentum/class_3/event/py_bias": 0.1788303256034851, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.869341850280762, "metrics/particle/momentum/class_3/event/pz_bias": 0.352527379989624, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.809297561645508, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02191113866865635, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14406348764896393, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1419423222541809, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.057349566370248795, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18815159797668457, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010238908231258392, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7154157757759094, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.21889014542102814, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8851629495620728, "metrics/particle/momentum/class_4/accepted/samples": 293.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2570977210998535, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.372964382171631, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1170601844787598, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.029637098312378, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.12777778506278992, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.422189712524414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013888888992369175, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.893324613571167, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10776755958795547, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.5836784839630127, "metrics/particle/momentum/class_4/assigned/samples": 360.0, "metrics/particle/momentum/class_4/efficiency": 0.8049450516700745, "metrics/particle/momentum/class_4/event/energy_bias": -0.6266441941261292, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.252650022506714, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.551115870475769, "metrics/particle/momentum/class_4/event/px_bias": 0.1468748152256012, "metrics/particle/momentum/class_4/event/py_bias": 0.01655137725174427, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.371301293373108, "metrics/particle/momentum/class_4/event/pz_bias": 0.2124408781528473, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.716164588928223, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012934633530676365, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11807038635015488, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12102464586496353, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03822966665029526, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1538342386484146, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.020270269364118576, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.327515959739685, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.25101417303085327, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.6481642723083496, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0765494555234909, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4648750424385071, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.35932251811027527, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2491481900215149, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.069182388484478, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.8332186937332153, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03773584961891174, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7488207817077637, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5346511602401733, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.890460252761841, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.9308176040649414, "metrics/particle/momentum/class_5/event/energy_bias": 0.23746520280838013, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6017539501190186, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2102104425430298, "metrics/particle/momentum/class_5/event/px_bias": 0.0012120679020881653, "metrics/particle/momentum/class_5/event/py_bias": 0.21692614257335663, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8242606520652771, "metrics/particle/momentum/class_5/event/pz_bias": 0.1960403025150299, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.9104669094085693, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.291274756193161, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2926315665245056, "metrics/set_slot/angular_recall_all_dr0p10": 0.9652900099754333, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8700630068778992, "metrics/set_slot/soft_count_bias_mean": -0.5006558299064636}, "step_train_time": 6.040063142776489, "step_valid_time": 63482.94114136696, "step_total_time": 63488.98120498657, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0079198544985521, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09905147090456179, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2910, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9613478691774033, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9603960396039604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9608717186726102, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039603960396039604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9573835480673935, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9564356435643564, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9569093610698366, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9613478691774033, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035227519645302, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10013339549687916, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2168, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9584438549955792, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.95929203539823, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9588677576293675, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04070796460176991, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571175950486295, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9579646017699115, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9575409111012826, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9584438549955792, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030389848478942, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09926277614663148, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1220, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9472049689440993, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9435421500386698, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9453700116234017, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05645784996133024, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9440993788819876, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9404485692188709, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9422704378148005, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9472049689440993} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_6000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..e839f659a7f812e54729e313e0d59479c13dabc4 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28425241614553726, "train": {"Classification_binary": 0.34123456478118896, "Cardinality": 0.25002264976501465, "Classification": 0.22660201787948608, "Regression_pt": 0.32119378447532654, "Regression_eta": 0.00479914341121912, "Regression_sin_phi": 0.0024189623072743416, "Regression_cos_phi": 0.0024442460853606462, "Regression_energy": 0.31871140003204346, "HitGrouping": 0.22835804522037506, "Auxiliary": 0.4061705768108368, "Total": 2.1979730129241943, "Total_unweighted": 1.2174041189718992}, "train_group": {}, "valid": {"Classification_binary": 0.31061355272928876, "Cardinality": 0.19724369049072266, "Classification": 0.21120697259902954, "Regression_pt": 0.33001871903737384, "Regression_eta": 0.004265340355535348, "Regression_sin_phi": 0.0022592234114805856, "Regression_cos_phi": 0.0022747356755038104, "Regression_energy": 0.3275546630223592, "HitGrouping": 0.2243895928064982, "Auxiliary": 0.38317235310872394, "Total": 2.04817787806193, "Total_unweighted": 1.1881932068305712, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8061739802360535, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3279608190059662, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.39888542890548706, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.32759714126586914, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.30760011076927185, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.45253613591194153, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7635378241539001, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.33835312724113464, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3981701135635376, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.33535364270210266, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3091755509376526, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5106743574142456, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8212646842002869, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.32238081097602844, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.37181076407432556, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.321696937084198, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2992553412914276, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.42242535948753357, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8112307190895081, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.32560908794403076, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3828875422477722, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.32482054829597473, "metrics/hit_grouping/calorimeter/all/object_purity": 0.30283045768737793, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.44003039598464966, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025120796635746956, "metrics/hit_grouping/calorimeter/background_recall": 0.6230436563491821, "metrics/hit_grouping/calorimeter/completeness": 0.4258841276168823, "metrics/hit_grouping/calorimeter/energy_completeness": 0.45016375184059143, "metrics/hit_grouping/calorimeter/energy_purity": 0.44470563530921936, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5227757096290588, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.841291606426239, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.32703885436058044, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3097641170024872, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.32859793305397034, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.29983094334602356, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.374798446893692, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.798065721988678, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.34500232338905334, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.40550702810287476, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.34246084094047546, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3331185281276703, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4770742356777191, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.761320948600769, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.30237317085266113, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5140942931175232, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.29844218492507935, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.27761635184288025, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.539501965045929, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7759125232696533, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3507806360721588, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4581127464771271, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.35136839747428894, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3415937125682831, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5159763097763062, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7576159238815308, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5106087923049927, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6127681732177734, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5032314658164978, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.49681755900382996, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5039246678352356, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.856060802936554, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2748749256134033, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2725994288921356, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2734498679637909, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.23633427917957306, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3502075672149658, "metrics/hit_grouping/calorimeter/purity": 0.41932380199432373, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8990396857261658, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05997202545404434, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13846154510974884, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.058278393000364304, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.018935825675725937, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04307692125439644, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8699855804443359, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.0739259421825409, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10889395326375961, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.07341926544904709, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.05377287045121193, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.243443563580513, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8059797883033752, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3457466959953308, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4038562476634979, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3449612259864807, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.32320618629455566, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.45979800820350647, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6535953879356384, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4604004919528961, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.34040337800979614, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.34670618176460266, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7157800197601318, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7445391416549683, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.4223219156265259, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4235806167125702, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.38974788784980774, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5389792323112488, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5902584791183472, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4628652334213257, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.27506405115127563, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2786747217178345, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7769551873207092, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6232706904411316, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.45867183804512024, "metrics/hit_grouping/tracker/all/object_completeness": 0.308223694562912, "metrics/hit_grouping/tracker/all/object_purity": 0.3095319867134094, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7375779151916504, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005959915462881327, "metrics/hit_grouping/tracker/background_recall": 0.03903334587812424, "metrics/hit_grouping/tracker/completeness": 0.3290950357913971, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5664798021316528, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6385106444358826, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4680637717247009, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3765909671783447, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.37647080421447754, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7288382649421692, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5989561676979065, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44831526279449463, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2555583715438843, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.2735855281352997, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7675613760948181, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6152740716934204, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4488268494606018, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2124146819114685, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.199130117893219, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7251213788986206, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6291996836662292, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4750179648399353, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.29753708839416504, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.300819993019104, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7320272922515869, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6549387574195862, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.48069241642951965, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3378761112689972, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3733733892440796, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6930758953094482, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6099782586097717, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.43350502848625183, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.31693151593208313, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3096557557582855, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7526340484619141, "metrics/hit_grouping/tracker/purity": 0.3140198290348053, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8783926367759705, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.016507385298609734, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03629135340452194, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.037998031824827194, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07775846868753433, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6502668261528015, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.17314670979976654, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.12378291040658951, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.11913269758224487, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5878693461418152, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5855298638343811, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5512661337852478, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3661762773990631, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3681088984012604, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8442831039428711, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4920317232608795, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.39705225825309753, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.41676065325737, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5559705495834351, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.006304370239377022, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.025165824219584465, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.049232482910156, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.19395865499973297, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.06351884454488754, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6159279942512512, "metrics/particle/class_1/pid_accuracy": 0.9131356477737427, "metrics/particle/class_2/efficiency": 0.4096926152706146, "metrics/particle/class_2/pid_accuracy": 0.39457830786705017, "metrics/particle/class_3/efficiency": 0.5201982259750366, "metrics/particle/class_3/pid_accuracy": 0.896857500076294, "metrics/particle/class_4/efficiency": 0.5168431401252747, "metrics/particle/class_4/pid_accuracy": 0.3165735602378845, "metrics/particle/class_5/efficiency": 0.6359999775886536, "metrics/particle/class_5/pid_accuracy": 0.3050314486026764, "metrics/particle/count/bias_mean": -7.575520992279053, "metrics/particle/count/mae": 8.553385734558105, "metrics/particle/count/prediction_mean": 62.48046875, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.10351047664880753, "metrics/particle/event/energy_response_mean": 0.9947946667671204, "metrics/particle/event/met_abs_error": 11.348404884338379, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0915459617972374, "metrics/particle/event/scalar_pt_response_mean": 1.017907977104187, "metrics/particle/event/vector_pt_closure": 0.0884719118475914, "metrics/particle/matched/delta_eta_abs_mean": 0.020184945315122604, "metrics/particle/matched/delta_phi_abs_mean": 0.019875656813383102, "metrics/particle/matched/delta_r_mean": 0.031653013080358505, "metrics/particle/matched/energy_relative_abs_mean": 0.19453929364681244, "metrics/particle/matched/pid_accuracy": 0.8663415908813477, "metrics/particle/matched/pt_relative_abs_mean": 0.1909787654876709, "metrics/particle/matching/duplicate_fraction": 0.11449411511421204, "metrics/particle/matching/efficiency": 0.5565117001533508, "metrics/particle/matching/f1": 0.5883208513259888, "metrics/particle/matching/purity": 0.6239866614341736, "metrics/particle/matching/target_coverage_dr0p05": 0.49097633361816406, "metrics/particle/matching/target_coverage_dr0p10": 0.64211106300354, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3467278778553009, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6147705316543579, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.030436111614108086, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1918732076883316, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.18874497711658478, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.014396349899470806, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22927047312259674, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014953271485865116, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.2131633758544922, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08791208267211914, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.775932550430298, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5885.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5288548469543457, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.145481824874878, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.748561382293701, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.5548765659332275, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2912546694278717, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.67610740661621, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.029224345460534096, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8461183309555054, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.18056263029575348, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.616573810577393, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9102.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5381309390068054, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6617432832717896, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.088561058044434, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.9160380363464355, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.653642475605011, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.320382297039032, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.168169975280762, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.730412483215332, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.937589645385742, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029046067968010902, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1912926584482193, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19147224724292755, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.004955498967319727, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23377174139022827, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.043090637773275375, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.2494466304779053, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07365880906581879, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.312269687652588, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1346.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6126257181167603, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7965449094772339, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.09906530380249, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.931741237640381, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29772835969924927, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.829465866088867, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0637989342212677, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.03169584274292, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.3951667249202728, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.50949764251709, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2069.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.44554781913757324, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.79383659362793, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.05223560333252, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7695763111114502, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.24698525667190552, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0773531049489975, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.824195861816406, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.748018264770508, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.261516571044922, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03221141919493675, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19564487040042877, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19264079630374908, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.022155186161398888, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23264332115650177, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002075516851618886, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.41961339116096497, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03509891778230667, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.087441086769104, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12527.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4954645037651062, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.066535472869873, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.425558567047119, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.22882342338562, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3102697432041168, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.257364273071289, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011828755028545856, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9262279272079468, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.07433456182479858, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1297800540924072, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20205.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5843634605407715, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.2056097984313965, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.752452850341797, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.526235580444336, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.286550521850586, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6096994876861572, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.1572465896606445, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.6174143552780151, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.502541542053223, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.031466998159885406, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19422495365142822, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19140078127384186, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.017997289076447487, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23172122240066528, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00870533473789692, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7806316018104553, "metrics/particle/momentum/all/accepted/pz_bias": 0.05345639958977699, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.228774309158325, "metrics/particle/momentum/all/accepted/samples": 19758.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5128766894340515, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.71559739112854, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.6296145915985107, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.435703754425049, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.303926557302475, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.517890930175781, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.020302141085267067, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.3319220542907715, "metrics/particle/momentum/all/assigned/pz_bias": 0.12630708515644073, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.9095871448516846, "metrics/particle/momentum/all/assigned/samples": 31376.0, "metrics/particle/momentum/all/efficiency": 0.5582302212715149, "metrics/particle/momentum/all/event/energy_bias": -1.249969244003296, "metrics/particle/momentum/all/event/p3_error_mean": 28.58055877685547, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.406030654907227, "metrics/particle/momentum/all/event/px_bias": 2.6932077407836914, "metrics/particle/momentum/all/event/py_bias": 0.8527287244796753, "metrics/particle/momentum/all/event/pz_abs_mean": 21.177536010742188, "metrics/particle/momentum/all/event/pz_bias": 6.095844745635986, "metrics/particle/momentum/all/event/pz_residual_rms": 33.93435287475586, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.038209762424230576, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21002474427223206, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.20664259791374207, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.017086349427700043, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2462829351425171, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.009891388937830925, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8840714693069458, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.07630719989538193, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.219822406768799, "metrics/particle/momentum/class_1/accepted/samples": 10312.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3578433096408844, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.272459626197815, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.3402687311172485, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.1275672912597656, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.260886549949646, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.476130485534668, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.017444510012865067, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.2559086084365845, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.17783068120479584, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.635890007019043, "metrics/particle/momentum/class_1/assigned/samples": 15363.0, "metrics/particle/momentum/class_1/efficiency": 0.6179660558700562, "metrics/particle/momentum/class_1/event/energy_bias": 18.35967254638672, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.16253662109375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.55277156829834, "metrics/particle/momentum/class_1/event/px_bias": 4.275173187255859, "metrics/particle/momentum/class_1/event/py_bias": -0.03510048985481262, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.956235885620117, "metrics/particle/momentum/class_1/event/pz_bias": 5.020413875579834, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.42832374572754, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03811993822455406, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22867538034915924, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22448354959487915, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07798630744218826, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2643822431564331, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031198685988783836, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8657695055007935, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04375293105840683, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.2282819747924805, "metrics/particle/momentum/class_2/accepted/samples": 1218.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6017066240310669, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7114219665527344, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.847571611404419, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4763162434101105, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29473185539245605, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.823549747467041, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04841006174683571, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5110082626342773, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.31298840045928955, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.522649765014648, "metrics/particle/momentum/class_2/assigned/samples": 2107.0, "metrics/particle/momentum/class_2/efficiency": 0.40913671255111694, "metrics/particle/momentum/class_2/event/energy_bias": -17.989402770996094, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.635887145996094, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.147905349731445, "metrics/particle/momentum/class_2/event/px_bias": 0.5134040117263794, "metrics/particle/momentum/class_2/event/py_bias": -1.4147361516952515, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.965572357177734, "metrics/particle/momentum/class_2/event/pz_bias": 1.46050226688385, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.6404972076416, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02160467766225338, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1678749918937683, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1660192757844925, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.01019451767206192, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2048070728778839, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003023812547326088, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.439153790473938, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.005680349189788103, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.6383814811706543, "metrics/particle/momentum/class_3/accepted/samples": 7937.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6792436838150024, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.641720294952393, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.356204986572266, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.212246417999268, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3535841703414917, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.422622680664062, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018292682245373726, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2013345956802368, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03528626635670662, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.865471363067627, "metrics/particle/momentum/class_3/assigned/samples": 13448.0, "metrics/particle/momentum/class_3/efficiency": 0.5219306945800781, "metrics/particle/momentum/class_3/event/energy_bias": -1.1448723077774048, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.900869369506836, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.91730785369873, "metrics/particle/momentum/class_3/event/px_bias": -1.546795129776001, "metrics/particle/momentum/class_3/event/py_bias": 0.8906217813491821, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.061939239501953, "metrics/particle/momentum/class_3/event/pz_bias": -0.5316849946975708, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.205299377441406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.034871894866228104, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2044718712568283, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19980962574481964, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028345556929707527, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23759867250919342, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.021052632480859756, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.6425334215164185, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.9305975437164307, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.1000661849975586, "metrics/particle/momentum/class_4/accepted/samples": 190.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5550101399421692, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.197508811950684, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.621315956115723, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.469715118408203, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.372307687997818, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.895736694335938, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02769230678677559, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.942491888999939, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5457950830459595, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.454500198364258, "metrics/particle/momentum/class_4/assigned/samples": 325.0, "metrics/particle/momentum/class_4/efficiency": 0.5219780206680298, "metrics/particle/momentum/class_4/event/energy_bias": 0.7623262405395508, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.232048511505127, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.626835346221924, "metrics/particle/momentum/class_4/event/px_bias": -0.23233658075332642, "metrics/particle/momentum/class_4/event/py_bias": 0.7075104713439941, "metrics/particle/momentum/class_4/event/pz_abs_mean": 3.0572094917297363, "metrics/particle/momentum/class_4/event/pz_bias": 0.2195395827293396, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.259202003479004, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.031424280256032944, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.21704299747943878, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21503913402557373, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0879252478480339, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.25633230805397034, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.039603959769010544, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.3467488288879395, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05819651111960411, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.900970458984375, "metrics/particle/momentum/class_5/accepted/samples": 101.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08895134180784225, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.34416115283966064, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.34757882356643677, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.020520329475402832, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.23308271169662476, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4904605746269226, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09022556245326996, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.145193576812744, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.6043716073036194, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.028764724731445, "metrics/particle/momentum/class_5/assigned/samples": 133.0, "metrics/particle/momentum/class_5/efficiency": 0.6352201104164124, "metrics/particle/momentum/class_5/event/energy_bias": -1.2376937866210938, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.141148090362549, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.1513025760650635, "metrics/particle/momentum/class_5/event/px_bias": -0.3162378966808319, "metrics/particle/momentum/class_5/event/py_bias": 0.7044329643249512, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.1978797912597656, "metrics/particle/momentum/class_5/event/pz_bias": -0.07292560487985611, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.739223003387451, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6098346710205078, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6118422746658325, "metrics/set_slot/angular_recall_all_dr0p10": 0.839804470539093, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6621564030647278, "metrics/set_slot/soft_count_bias_mean": 0.04967741295695305}, "step_train_time": 5.3051416873931885, "step_valid_time": 18246.629219055176, "step_total_time": 18251.934361219406, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0482290602728206, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24826503515440557, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2995, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2647, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2524, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8744631648496861, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8838063439065108, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.879109930255729, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11619365609348915, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8338288734720846, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.842737896494157, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8382597143806044, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8744631648496861, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0237762768292593, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22737768554264526, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2268, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2008, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1969, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.887709991158267, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8853615520282186, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8865342163355409, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1146384479717813, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8704686118479222, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8681657848324515, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8693156732891832, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.887709991158267, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0274307417472641, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21523723927799066, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1256, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1107, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1079, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.859472049689441, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8813694267515924, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8702830188679245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11863057324840764, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8377329192546584, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8590764331210191, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8482704402515723, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.859472049689441} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_7000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..d5141a9dfcf9a9a37ed928c614df6b664e3e4607 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.3316278188364601, "train": {"Classification_binary": 0.35228100419044495, "Cardinality": 0.23500049114227295, "Classification": 0.23497527837753296, "Regression_pt": 0.3160865902900696, "Regression_eta": 0.005144247319549322, "Regression_sin_phi": 0.0026420443318784237, "Regression_cos_phi": 0.0026538169477134943, "Regression_energy": 0.31550779938697815, "HitGrouping": 0.2221004068851471, "Auxiliary": 0.39546552300453186, "Total": 2.2195987701416016, "Total_unweighted": 1.2292907808441669}, "train_group": {}, "valid": {"Classification_binary": 0.3017731308937073, "Cardinality": 0.19070412715276083, "Classification": 0.20041908820470175, "Regression_pt": 0.30323614676793414, "Regression_eta": 0.004162034951150417, "Regression_sin_phi": 0.0021765511482954025, "Regression_cos_phi": 0.00217087489242355, "Regression_energy": 0.29997313022613525, "HitGrouping": 0.21734821796417236, "Auxiliary": 0.3616149425506592, "Total": 1.9521293640136719, "Total_unweighted": 1.1139109570843477, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8111310601234436, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3483109474182129, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37243789434432983, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3464111387729645, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.33547425270080566, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.44148483872413635, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7769201397895813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3537423610687256, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.38393765687942505, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.35245081782341003, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.33110567927360535, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.48729243874549866, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8242430090904236, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.34005385637283325, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35017043352127075, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.33916881680488586, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.32856228947639465, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.42779672145843506, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8157789707183838, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3439387381076813, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.36036473512649536, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.34269070625305176, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3310146629810333, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4375379979610443, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025018781423568726, "metrics/hit_grouping/calorimeter/background_recall": 0.6209400296211243, "metrics/hit_grouping/calorimeter/completeness": 0.45094794034957886, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4781194031238556, "metrics/hit_grouping/calorimeter/energy_purity": 0.4722132980823517, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4794315993785858, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.836743950843811, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.34880968928337097, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2937593460083008, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.34976908564567566, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3344745635986328, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.38518962264060974, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8065945506095886, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3641860783100128, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38731199502944946, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36158475279808044, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.36147844791412354, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4653080999851227, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7809739708900452, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.31252890825271606, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4732650816440582, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3080187439918518, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2919451892375946, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.519403338432312, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.791002631187439, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3706282675266266, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4401121139526367, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.37121859192848206, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3656465411186218, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4947991371154785, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8026464581489563, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5509822368621826, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6091051697731018, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5461398959159851, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4815245270729065, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4385138750076294, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8440956473350525, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2887519299983978, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24280166625976562, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2859874367713928, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.27452346682548523, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3758537471294403, "metrics/hit_grouping/calorimeter/purity": 0.44395509362220764, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8887820243835449, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.057812564074993134, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1230769231915474, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05758974701166153, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.015011993236839771, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05076923221349716, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8480991125106812, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.0869874581694603, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09863170236349106, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.08649835735559464, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.07002439349889755, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29988598823547363, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8123641014099121, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3648160696029663, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.38047611713409424, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.36350250244140625, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.35276108980178833, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4536988437175751, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.660507082939148, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.47160789370536804, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.36013588309288025, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.36970096826553345, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7130902409553528, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7450757026672363, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.40830060839653015, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4271392524242401, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.40071845054626465, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5451486110687256, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5966064929962158, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4725525677204132, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.29756399989128113, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3142866790294647, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7723872065544128, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6293073296546936, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.46684300899505615, "metrics/hit_grouping/tracker/all/object_completeness": 0.32825374603271484, "metrics/hit_grouping/tracker/all/object_purity": 0.3390807509422302, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7345078587532043, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0046209171414375305, "metrics/hit_grouping/tracker/background_recall": 0.023159390315413475, "metrics/hit_grouping/tracker/completeness": 0.347768634557724, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5640120506286621, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6422297358512878, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4625968039035797, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3921361565589905, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.40482765436172485, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7305694818496704, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6077280640602112, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.47154006361961365, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.28713157773017883, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.30834296345710754, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.758233368396759, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6235494017601013, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4712783098220825, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.230135977268219, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2257906198501587, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7180420756340027, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6339349746704102, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4937095642089844, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.32251572608947754, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3381561040878296, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7320272922515869, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.670077383518219, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4993342161178589, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.36933624744415283, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.39157408475875854, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6684420704841614, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6161099076271057, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.42612969875335693, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.32850512862205505, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.33105504512786865, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7493561506271362, "metrics/hit_grouping/tracker/purity": 0.3313916027545929, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8805755376815796, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.017376193776726723, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03853115811944008, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04059196263551712, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0842745453119278, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6475615501403809, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.20114049315452576, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.15168364346027374, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.14909471571445465, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6210471987724304, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5930857062339783, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5581724643707275, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3877275288105011, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.40127333998680115, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8357830047607422, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.48398706316947937, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.41554558277130127, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.43997722864151, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5410975813865662, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0017402907833456993, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.010292856022715569, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.16945457458496, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2039748728275299, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.07353506982326508, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6218836307525635, "metrics/particle/class_1/pid_accuracy": 0.9338212013244629, "metrics/particle/class_2/efficiency": 0.39185550808906555, "metrics/particle/class_2/pid_accuracy": 0.44059547781944275, "metrics/particle/class_3/efficiency": 0.5185463428497314, "metrics/particle/class_3/pid_accuracy": 0.9069960117340088, "metrics/particle/class_4/efficiency": 0.5332050323486328, "metrics/particle/class_4/pid_accuracy": 0.28700360655784607, "metrics/particle/class_5/efficiency": 0.6639999747276306, "metrics/particle/class_5/pid_accuracy": 0.34036144614219666, "metrics/particle/count/bias_mean": -9.934245109558105, "metrics/particle/count/mae": 10.297526359558105, "metrics/particle/count/prediction_mean": 60.12174606323242, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.10100206732749939, "metrics/particle/event/energy_response_mean": 0.9514226913452148, "metrics/particle/event/met_abs_error": 11.179156303405762, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08695155382156372, "metrics/particle/event/scalar_pt_response_mean": 0.9661910533905029, "metrics/particle/event/vector_pt_closure": 0.08523780107498169, "metrics/particle/matched/delta_eta_abs_mean": 0.0214177668094635, "metrics/particle/matched/delta_phi_abs_mean": 0.019590696319937706, "metrics/particle/matched/delta_r_mean": 0.03236938267946243, "metrics/particle/matched/energy_relative_abs_mean": 0.19159933924674988, "metrics/particle/matched/pid_accuracy": 0.8850283622741699, "metrics/particle/matched/pt_relative_abs_mean": 0.18751773238182068, "metrics/particle/matching/duplicate_fraction": 0.09939683973789215, "metrics/particle/matching/efficiency": 0.5574038624763489, "metrics/particle/matching/f1": 0.5999410152435303, "metrics/particle/matching/purity": 0.6495067477226257, "metrics/particle/matching/target_coverage_dr0p05": 0.4943869411945343, "metrics/particle/matching/target_coverage_dr0p10": 0.6406520009040833, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3533167243003845, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6183484196662903, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03135044500231743, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18961936235427856, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1852450668811798, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014041362330317497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22594554722309113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01436591800302267, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.144606351852417, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.11453618854284286, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7380435466766357, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6056.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4267820119857788, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.585336685180664, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6742138862609863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4872465133666992, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2539035975933075, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.214211463928223, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025910839438438416, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.6536273956298828, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.17981256544589996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.068063259124756, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8838.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5537673830986023, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.2734272480010986, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.586703300476074, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.676239967346191, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.263630747795105, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.889347493648529, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.821442604064941, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7586618661880493, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.781150817871094, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030658291652798653, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19394728541374207, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18711534142494202, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02137571945786476, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2276589423418045, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04078857973217964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.9667243957519531, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.32202717661857605, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.375798225402832, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1471.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5047567486763, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0024709701538086, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3683087825775146, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1856493949890137, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24737344682216644, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 13.3207368850708, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06255969405174255, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.789034366607666, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2980823516845703, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.795370578765869, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2094.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4869248569011688, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.569563865661621, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.812809944152832, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8820295333862305, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0931597650051117, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.054114438593387604, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.602787017822266, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6235442757606506, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.149383544921875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03284767270088196, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19160552322864532, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18820400536060333, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011328820139169693, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2294078767299652, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0013849287061020732, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3938799500465393, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006877359468489885, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0426650047302246, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12275.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4834947884082794, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.3109257221221924, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.8278532028198242, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.635304570198059, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.29268166422843933, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.177205085754395, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008102979511022568, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8046727180480957, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005350450519472361, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.707761526107788, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19499.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5726081132888794, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.7118749618530273, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.827549934387207, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.425130844116211, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.388190269470215, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6888875365257263, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.441615581512451, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.028298228979110718, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.842955589294434, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03222714364528656, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1911720484495163, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18721818923950195, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012904728762805462, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.228224515914917, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008281991817057133, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7403122186660767, "metrics/particle/momentum/all/accepted/pz_bias": 0.054687075316905975, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0950067043304443, "metrics/particle/momentum/all/accepted/samples": 19802.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4684869050979614, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.010158061981201, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.820421576499939, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.6301743984222412, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.27830174565315247, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.908243179321289, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017022116109728813, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1877793073654175, "metrics/particle/momentum/all/assigned/pz_bias": 0.07616234570741653, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.561162233352661, "metrics/particle/momentum/all/assigned/samples": 30431.0, "metrics/particle/momentum/all/efficiency": 0.5594733357429504, "metrics/particle/momentum/all/event/energy_bias": -14.554864883422852, "metrics/particle/momentum/all/event/p3_error_mean": 28.492961883544922, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.139440536499023, "metrics/particle/momentum/all/event/px_bias": 3.558661460876465, "metrics/particle/momentum/all/event/py_bias": 0.25457435846328735, "metrics/particle/momentum/all/event/pz_abs_mean": 21.138980865478516, "metrics/particle/momentum/all/event/pz_bias": 0.16341593861579895, "metrics/particle/momentum/all/event/pz_residual_rms": 32.30929946899414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03886372223496437, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.20074068009853363, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.19587060809135437, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002351386472582817, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.23641443252563477, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.008440437726676464, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8206012845039368, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.08460766077041626, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.120457887649536, "metrics/particle/momentum/class_1/accepted/samples": 10426.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3310142457485199, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0095511674880981, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.0343194007873535, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8320112228393555, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.24053318798542023, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.661790370941162, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014258240349590778, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1105871200561523, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.10336358100175858, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.041771173477173, "metrics/particle/momentum/class_1/assigned/samples": 15079.0, "metrics/particle/momentum/class_1/efficiency": 0.6247977614402771, "metrics/particle/momentum/class_1/event/energy_bias": 14.895057678222656, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.442270278930664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.322905540466309, "metrics/particle/momentum/class_1/event/px_bias": 1.840679407119751, "metrics/particle/momentum/class_1/event/py_bias": -0.22503390908241272, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.840337753295898, "metrics/particle/momentum/class_1/event/pz_bias": 2.4286561012268066, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.96907615661621, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036176443099975586, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.218933567404747, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2164260745048523, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07049071043729782, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.253967821598053, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03723404183983803, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8833763599395752, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.19586937129497528, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.128974437713623, "metrics/particle/momentum/class_2/accepted/samples": 1128.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5633748769760132, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5821311473846436, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7617641687393188, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.35844701528549194, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29466596245765686, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.83217191696167, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05230450630187988, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5533523559570312, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.12678807973861694, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.097651958465576, "metrics/particle/momentum/class_2/assigned/samples": 1931.0, "metrics/particle/momentum/class_2/efficiency": 0.37890493869781494, "metrics/particle/momentum/class_2/event/energy_bias": -21.9973201751709, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.35266876220703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.785797119140625, "metrics/particle/momentum/class_2/event/px_bias": 1.463273286819458, "metrics/particle/momentum/class_2/event/py_bias": -0.39941301941871643, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.942432403564453, "metrics/particle/momentum/class_2/event/pz_bias": -0.33865538239479065, "metrics/particle/momentum/class_2/event/pz_residual_rms": 28.55075454711914, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0228671133518219, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17430871725082397, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17111331224441528, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03945671021938324, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21215909719467163, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0033975085243582726, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4362361431121826, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.011384395882487297, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4729946851730347, "metrics/particle/momentum/class_3/accepted/samples": 7947.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6163458824157715, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.3937950134277344, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.9007608890533447, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.758657217025757, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3195033669471741, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.734129905700684, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014112748205661774, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0421273708343506, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.024024317041039467, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.557875633239746, "metrics/particle/momentum/class_3/assigned/samples": 12967.0, "metrics/particle/momentum/class_3/efficiency": 0.5225882530212402, "metrics/particle/momentum/class_3/event/energy_bias": -3.9587414264678955, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.273039817810059, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.439781188964844, "metrics/particle/momentum/class_3/event/px_bias": 0.3959802985191345, "metrics/particle/momentum/class_3/event/py_bias": 0.3372512459754944, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.669933319091797, "metrics/particle/momentum/class_3/event/pz_bias": -2.71781587600708, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.34848976135254, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.035873301327228546, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17855900526046753, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18093793094158173, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02184167504310608, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22471316158771515, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010309278033673763, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2917027473449707, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.7065942287445068, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.472238063812256, "metrics/particle/momentum/class_4/accepted/samples": 194.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5296905040740967, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.318084478378296, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.049741268157959, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.831776738166809, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.33018869161605835, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.369354248046875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.012578615918755531, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.5456470251083374, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.7447048425674438, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.629394769668579, "metrics/particle/momentum/class_4/assigned/samples": 318.0, "metrics/particle/momentum/class_4/efficiency": 0.5329670310020447, "metrics/particle/momentum/class_4/event/energy_bias": -1.8565438985824585, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.5937297344207764, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.581960678100586, "metrics/particle/momentum/class_4/event/px_bias": 0.22950071096420288, "metrics/particle/momentum/class_4/event/py_bias": 0.23801937699317932, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.094726324081421, "metrics/particle/momentum/class_4/event/pz_bias": 0.7852374911308289, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.501338005065918, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03249790146946907, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24147416651248932, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.24373690783977509, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06787236034870148, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2806817889213562, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04672897234559059, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.451103448867798, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.6238700151443481, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.710404396057129, "metrics/particle/momentum/class_5/accepted/samples": 107.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12269558757543564, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.584639847278595, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4690580368041992, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11589132249355316, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.18382352590560913, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.3102952241897583, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.11029411852359772, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.407813310623169, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2506640553474426, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.7937469482421875, "metrics/particle/momentum/class_5/assigned/samples": 136.0, "metrics/particle/momentum/class_5/efficiency": 0.6729559898376465, "metrics/particle/momentum/class_5/event/energy_bias": -1.6373170614242554, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.97847843170166, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.1135454177856445, "metrics/particle/momentum/class_5/event/px_bias": -0.3707726299762726, "metrics/particle/momentum/class_5/event/py_bias": 0.30375078320503235, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.0468215942382812, "metrics/particle/momentum/class_5/event/pz_bias": 0.005993686616420746, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.231409549713135, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5984908938407898, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.602374792098999, "metrics/set_slot/angular_recall_all_dr0p10": 0.8451852202415466, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6600375175476074, "metrics/set_slot/soft_count_bias_mean": -0.7730655670166016}, "step_train_time": 5.5758771896362305, "step_valid_time": 21357.50169467926, "step_total_time": 21363.077572345734, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0264788561240945, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22262573850856726, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3007, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2698, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2604, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8913115295672283, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8972397738609911, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8942658269804441, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10276022613900898, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8602576808721506, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.865979381443299, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8631090487238979, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8913115295672283, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9770365185061739, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21449397704137307, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2258, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2026, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2007, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8956675508399646, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8972542072630647, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8964601769911504, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10274579273693533, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8872679045092838, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8888396811337467, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8880530973451327, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8956675508399646, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9699031778321097, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21561570064449564, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1254, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1114, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1087, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8649068322981367, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8883572567783095, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8764752163650669, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11164274322169059, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.843944099378882, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8668261562998405, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8552321007081038, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8649068322981367} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_8000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..f8dc26d27face5ce8b7ce62728f243d411a3ddf6 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.37900322152738297, "train": {"Classification_binary": 0.32004860043525696, "Cardinality": 0.19156868755817413, "Classification": 0.19507679343223572, "Regression_pt": 0.28116506338119507, "Regression_eta": 0.004543783143162727, "Regression_sin_phi": 0.0021768989972770214, "Regression_cos_phi": 0.002195653971284628, "Regression_energy": 0.27888020873069763, "HitGrouping": 0.21406808495521545, "Auxiliary": 0.3560567796230316, "Total": 1.9491093158721924, "Total_unweighted": 1.0840870020911098}, "train_group": {}, "valid": {"Classification_binary": 0.2919641137123108, "Cardinality": 0.18567858139673868, "Classification": 0.1792048215866089, "Regression_pt": 0.2750618855158488, "Regression_eta": 0.0037007244924704232, "Regression_sin_phi": 0.002027861773967743, "Regression_cos_phi": 0.0020624883472919464, "Regression_energy": 0.2732822100321452, "HitGrouping": 0.21046201388041177, "Auxiliary": 0.3375240166982015, "Total": 1.8324516614278157, "Total_unweighted": 1.0273041054606438, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8117880821228027, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.37929481267929077, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3933125436306, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3783324658870697, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.35591796040534973, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4373287856578827, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7656165361404419, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.38465529680252075, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4134191870689392, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3823060691356659, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3567923605442047, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4981362223625183, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8206565976142883, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3751038610935211, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3586922883987427, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.37452876567840576, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3566376566886902, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4223737120628357, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8128659129142761, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3773092031478882, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.37474164366722107, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3764503598213196, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3564199209213257, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4339817762374878, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025385739281773567, "metrics/hit_grouping/calorimeter/background_recall": 0.6249128580093384, "metrics/hit_grouping/calorimeter/completeness": 0.47142335772514343, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4888613820075989, "metrics/hit_grouping/calorimeter/energy_purity": 0.48300257325172424, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.47800102829933167, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8349883556365967, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.38042399287223816, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2998507022857666, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3816831409931183, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.35960203409194946, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3845924139022827, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.805053174495697, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4003711938858032, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3999272286891937, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3987225592136383, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.38600999116897583, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4564531743526459, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7741813659667969, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3466835021972656, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5070155262947083, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.34216466546058655, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3188488781452179, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5151055455207825, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7938181161880493, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.40851032733917236, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.45107442140579224, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4103306531906128, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3924024999141693, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.47262534499168396, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7772113084793091, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5356605052947998, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6122449040412903, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5320268273353577, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5061812400817871, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4772370457649231, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8379059433937073, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32350289821624756, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2622874081134796, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3201189339160919, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2985725402832031, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3869023621082306, "metrics/hit_grouping/calorimeter/purity": 0.46428772807121277, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.893743634223938, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06500788778066635, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12153846025466919, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0652967020869255, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.021044408902525902, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04615384712815285, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8348020315170288, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12093612551689148, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11003420501947403, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.12040948867797852, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.10983389616012573, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.31413909792900085, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8098803758621216, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3987112045288086, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.39536333084106445, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3978087604045868, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37775081396102905, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.44914087653160095, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6629963517189026, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.47280335426330566, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.37733685970306396, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3897296190261841, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7030783295631409, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7569458484649658, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.40044867992401123, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.44463518261909485, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.42559105157852173, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5227145552635193, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.599561333656311, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.469874769449234, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3095792829990387, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.32444629073143005, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.761597216129303, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.632865846157074, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4649537205696106, "metrics/hit_grouping/tracker/all/object_completeness": 0.3423696458339691, "metrics/hit_grouping/tracker/all/object_purity": 0.35359883308410645, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.722983181476593, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0055084992200136185, "metrics/hit_grouping/tracker/background_recall": 0.0319848358631134, "metrics/hit_grouping/tracker/completeness": 0.3611716628074646, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5435740351676941, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6499525904655457, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.46059226989746094, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.40732541680336, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4179636240005493, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7173576354980469, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6112978458404541, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4780125617980957, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3030363917350769, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.32718247175216675, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7487150430679321, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6178514361381531, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4607605040073395, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2399684637784958, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.23878508806228638, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7081310749053955, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6377367973327637, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48930624127388, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3399810791015625, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.35978248715400696, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7166606783866882, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6857576370239258, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5026631355285645, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.38726285099983215, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.39894554018974304, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6331557631492615, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.617219865322113, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4265979826450348, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.33758753538131714, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3375694155693054, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7470147609710693, "metrics/hit_grouping/tracker/purity": 0.34447064995765686, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8776291608810425, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.019113814458251, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0565338209271431, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0617629699409008, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.08601216226816177, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6471932530403137, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21358217298984528, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.17403368651866913, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.16364575922489166, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.602384626865387, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5979750752449036, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5541585683822632, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.40037772059440613, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4148835837841034, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8232690095901489, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.468758225440979, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6749999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4420839846134186, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.476667195558548, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5773977041244507, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0169691052287817, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04659290984272957, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.803314208984375, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2253938913345337, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.09495407342910767, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6780767440795898, "metrics/particle/class_1/pid_accuracy": 0.9365042448043823, "metrics/particle/class_2/efficiency": 0.41462868452072144, "metrics/particle/class_2/pid_accuracy": 0.46266233921051025, "metrics/particle/class_3/efficiency": 0.5410293340682983, "metrics/particle/class_3/pid_accuracy": 0.9191482663154602, "metrics/particle/class_4/efficiency": 0.587103009223938, "metrics/particle/class_4/pid_accuracy": 0.28688523173332214, "metrics/particle/class_5/efficiency": 0.7400000095367432, "metrics/particle/class_5/pid_accuracy": 0.38108107447624207, "metrics/particle/count/bias_mean": -8.645833015441895, "metrics/particle/count/mae": 9.182291984558105, "metrics/particle/count/prediction_mean": 61.41015625, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.1076577827334404, "metrics/particle/event/energy_response_mean": 0.9201157093048096, "metrics/particle/event/met_abs_error": 10.110360145568848, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08811057358980179, "metrics/particle/event/scalar_pt_response_mean": 0.9508377909660339, "metrics/particle/event/vector_pt_closure": 0.0807110071182251, "metrics/particle/matched/delta_eta_abs_mean": 0.019283363595604897, "metrics/particle/matched/delta_phi_abs_mean": 0.019455155357718468, "metrics/particle/matched/delta_r_mean": 0.0306839719414711, "metrics/particle/matched/energy_relative_abs_mean": 0.18241530656814575, "metrics/particle/matched/pid_accuracy": 0.8930101990699768, "metrics/particle/matched/pt_relative_abs_mean": 0.1809501200914383, "metrics/particle/matching/duplicate_fraction": 0.10484913736581802, "metrics/particle/matching/efficiency": 0.5962957739830017, "metrics/particle/matching/f1": 0.6355109810829163, "metrics/particle/matching/purity": 0.6802472472190857, "metrics/particle/matching/target_coverage_dr0p05": 0.5261974334716797, "metrics/particle/matching/target_coverage_dr0p10": 0.6655669808387756, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3887329697608948, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6551028490066528, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.029533620923757553, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18050611019134521, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17909720540046692, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.01978202909231186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22001005709171295, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014658232219517231, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.102143406867981, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05660470575094223, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7583425045013428, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6481.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.39236122369766235, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4297255277633667, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6754504442214966, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4676154851913452, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.22128039598464966, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.687690734863281, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.023644881322979927, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5199381113052368, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.18165619671344757, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6041572093963623, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8966.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5926298499107361, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -7.075690269470215, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.119030952453613, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.168841361999512, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4388953745365143, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6178194880485535, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.541365623474121, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.487856388092041, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.96278190612793, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028998613357543945, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19021075963974, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1869649887084961, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.044232647866010666, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22704236209392548, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04406130313873291, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.979394793510437, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04875529184937477, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.562978744506836, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1566.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4292820394039154, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0474309921264648, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.666234254837036, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4368648529052734, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2028571367263794, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.579662322998047, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06142857298254967, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6810076236724854, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06167479604482651, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.658544063568115, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2100.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5183714032173157, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.990753173828125, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.304354667663574, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.998084306716919, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09904768317937851, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04554402828216553, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.07017707824707, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6172503232955933, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.147001266479492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0313253216445446, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18112877011299133, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18062326312065125, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.009026511572301388, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2212349772453308, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0016747867921367288, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3854075074195862, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02593444287776947, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.084916114807129, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13136.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.453862726688385, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.3990416526794434, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.968000054359436, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7715479135513306, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2669046223163605, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.90853214263916, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008244934491813183, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7731459736824036, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06545010954141617, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.661526918411255, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19891.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6127722859382629, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -4.0826416015625, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.542254447937012, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.302204132080078, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.09044652432203293, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9852686524391174, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.889248371124268, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.541081190109253, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.932401657104492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.030605139210820198, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18160967528820038, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18062520027160645, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.014919886365532875, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22129619121551514, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008780625648796558, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7225338816642761, "metrics/particle/momentum/all/accepted/pz_bias": 0.029796497896313667, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1440517902374268, "metrics/particle/momentum/all/accepted/samples": 21183.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.43438273668289185, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.026613235473633, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.9306350946426392, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.728653073310852, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24934586882591248, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.423959732055664, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01631294935941696, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1188592910766602, "metrics/particle/momentum/all/assigned/pz_bias": 0.09885049611330032, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.36472749710083, "metrics/particle/momentum/all/assigned/samples": 30957.0, "metrics/particle/momentum/all/efficiency": 0.5984912514686584, "metrics/particle/momentum/all/event/energy_bias": -23.149084091186523, "metrics/particle/momentum/all/event/p3_error_mean": 27.32355308532715, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.838199615478516, "metrics/particle/momentum/all/event/px_bias": 0.24940118193626404, "metrics/particle/momentum/all/event/py_bias": 1.5575441122055054, "metrics/particle/momentum/all/event/pz_abs_mean": 20.914257049560547, "metrics/particle/momentum/all/event/pz_bias": 4.41168737411499, "metrics/particle/momentum/all/event/pz_residual_rms": 31.076488494873047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03674204275012016, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.18764302134513855, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18742910027503967, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0351300872862339, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2271209955215454, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.008877559565007687, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7808101773262024, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06095267832279205, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.1400225162506104, "metrics/particle/momentum/class_1/accepted/samples": 11377.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.29073160886764526, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8228852152824402, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.9253852963447571, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.6987531781196594, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.19042330980300903, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.739806175231934, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.013587666675448418, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.030785083770752, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11129686236381531, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8732540607452393, "metrics/particle/momentum/class_1/assigned/samples": 15308.0, "metrics/particle/momentum/class_1/efficiency": 0.6817882061004639, "metrics/particle/momentum/class_1/event/energy_bias": 3.076390266418457, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.903493881225586, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.242727279663086, "metrics/particle/momentum/class_1/event/px_bias": -0.26048749685287476, "metrics/particle/momentum/class_1/event/py_bias": 2.388211250305176, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.605020523071289, "metrics/particle/momentum/class_1/event/pz_bias": 5.142230987548828, "metrics/particle/momentum/class_1/event/pz_residual_rms": 23.338134765625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036510203033685684, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2241835743188858, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22337624430656433, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08506210148334503, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26292306184768677, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03784219175577164, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.973443865776062, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.057676225900650024, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.38468599319458, "metrics/particle/momentum/class_2/accepted/samples": 1242.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5265533328056335, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6485758423805237, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8444427251815796, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.45182090997695923, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2850024998188019, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.8702986240386963, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04982560873031616, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5440220832824707, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1803174912929535, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.190941333770752, "metrics/particle/momentum/class_2/assigned/samples": 2007.0, "metrics/particle/momentum/class_2/efficiency": 0.41719850897789, "metrics/particle/momentum/class_2/event/energy_bias": -19.877355575561523, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.353660583496094, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.092079162597656, "metrics/particle/momentum/class_2/event/px_bias": 1.2844655513763428, "metrics/particle/momentum/class_2/event/py_bias": 0.48600199818611145, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.163663864135742, "metrics/particle/momentum/class_2/event/pz_bias": -0.8019818067550659, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.158037185668945, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021116716787219048, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16645777225494385, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16439957916736603, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024826059117913246, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20522041618824005, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003157255705446005, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4111350476741791, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009166575036942959, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.402612328529358, "metrics/particle/momentum/class_3/accepted/samples": 8235.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5902461409568787, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.6190176010131836, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.253448486328125, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.1103110313415527, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3130216896533966, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.14928436279297, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013127940706908703, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.970447301864624, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06270460784435272, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.223581552505493, "metrics/particle/momentum/class_3/assigned/samples": 13178.0, "metrics/particle/momentum/class_3/efficiency": 0.5415269136428833, "metrics/particle/momentum/class_3/event/energy_bias": -2.895305633544922, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.025609970092773, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.348444938659668, "metrics/particle/momentum/class_3/event/px_bias": -0.42446911334991455, "metrics/particle/momentum/class_3/event/py_bias": -1.6205737590789795, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.410584449768066, "metrics/particle/momentum/class_3/event/pz_bias": -0.7089541554450989, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.285812377929688, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03399593383073807, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1715644747018814, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1710037887096405, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.006658261641860008, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21253925561904907, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01886792480945587, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.287394642829895, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.4717731773853302, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.117743730545044, "metrics/particle/momentum/class_4/accepted/samples": 212.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4578203558921814, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.398554563522339, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.0479507446289062, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.859616994857788, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2639751434326172, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.621142387390137, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.02484472095966339, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.5706987380981445, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.49632108211517334, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.079596519470215, "metrics/particle/momentum/class_4/assigned/samples": 322.0, "metrics/particle/momentum/class_4/efficiency": 0.5824176073074341, "metrics/particle/momentum/class_4/event/energy_bias": -1.5402981042861938, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.4473185539245605, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.342503070831299, "metrics/particle/momentum/class_4/event/px_bias": -0.17364804446697235, "metrics/particle/momentum/class_4/event/py_bias": -0.20770752429962158, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.174079418182373, "metrics/particle/momentum/class_4/event/pz_bias": 0.48779016733169556, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.511981010437012, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03286706283688545, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22765690088272095, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22466666996479034, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.11757978796958923, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2655213177204132, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0683760717511177, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.671085834503174, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.12969912588596344, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.175976753234863, "metrics/particle/momentum/class_5/accepted/samples": 117.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09994428604841232, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.37814030051231384, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.35697656869888306, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.014950077049434185, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15492957830429077, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6437201499938965, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.11267605423927307, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.2189950942993164, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.058791980147361755, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.86104154586792, "metrics/particle/momentum/class_5/assigned/samples": 142.0, "metrics/particle/momentum/class_5/efficiency": 0.7358490824699402, "metrics/particle/momentum/class_5/event/energy_bias": -1.9125165939331055, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.540532112121582, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.6566972732543945, "metrics/particle/momentum/class_5/event/px_bias": -0.17645971477031708, "metrics/particle/momentum/class_5/event/py_bias": 0.5116122961044312, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.9172765016555786, "metrics/particle/momentum/class_5/event/pz_bias": 0.2926022410392761, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.835330009460449, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5656416416168213, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5683391690254211, "metrics/set_slot/angular_recall_all_dr0p10": 0.8541437983512878, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6792836785316467, "metrics/set_slot/soft_count_bias_mean": -0.17500080168247223}, "step_train_time": 5.174543142318726, "step_valid_time": 24501.95475912094, "step_total_time": 24507.12930250168, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048349376559158, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20501106959529614, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2985, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2696, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2623, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8906508093822266, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9031825795644891, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8968729208250167, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09681742043551089, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8665345226296663, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8787269681742044, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8725881570192947, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8906508093822266, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.967562088984707, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20275030292281349, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2226, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2030, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2003, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8974358974358975, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9119496855345912, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9046345811051694, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0880503144654088, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.885499557913351, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8998203054806828, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.892602495543672, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8974358974358975, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9652650213351229, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20531699160865968, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1245, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1117, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1101, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8672360248447205, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8971887550200803, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8819581523884722, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10281124497991968, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8548136645962733, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8843373493975903, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8693249111725228, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8672360248447205} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_9000.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..797b84fdaf1218bbe431a1fd906604ced0cad7cb --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.42637862421830586, "train": {"Classification_binary": 0.32325246930122375, "Cardinality": 0.2352568358182907, "Classification": 0.15861614048480988, "Regression_pt": 0.26531338691711426, "Regression_eta": 0.0036221016198396683, "Regression_sin_phi": 0.0019258398097008467, "Regression_cos_phi": 0.001862778328359127, "Regression_energy": 0.2633375823497772, "HitGrouping": 0.20636539161205292, "Auxiliary": 0.35886144638061523, "Total": 1.8831294775009155, "Total_unweighted": 1.0179302988108248}, "train_group": {}, "valid": {"Classification_binary": 0.2762115995089213, "Cardinality": 0.166849156220754, "Classification": 0.15786372621854147, "Regression_pt": 0.2566426595052083, "Regression_eta": 0.0036647341524561248, "Regression_sin_phi": 0.0018730050263305504, "Regression_cos_phi": 0.0018649169554313023, "Regression_energy": 0.25421061118443805, "HitGrouping": 0.20465817054112753, "Auxiliary": 0.3195289174715678, "Total": 1.7142450014750164, "Total_unweighted": 0.9523312525513271, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8138895034790039, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4030000865459442, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3714933395385742, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.40439409017562866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.38471588492393494, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10587.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4319448471069336, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7683398127555847, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.41485854983329773, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.4046086072921753, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.41628459095954895, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.39280927181243896, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2951.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4913588762283325, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8352085947990417, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.39852988719940186, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35213303565979004, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3986426889896393, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.37599363923072815, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 19362.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3874083161354065, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8223503828048706, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.40143296122550964, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.36306992173194885, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.402075856924057, "metrics/hit_grouping/calorimeter/all/object_purity": 0.38030868768692017, "metrics/hit_grouping/calorimeter/all/object_samples": 32900.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4110638201236725, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.025610074400901794, "metrics/hit_grouping/calorimeter/background_recall": 0.6279541850090027, "metrics/hit_grouping/calorimeter/completeness": 0.48919323086738586, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5031698346138, "metrics/hit_grouping/calorimeter/energy_purity": 0.4971451759338379, "metrics/hit_grouping/calorimeter/hit_samples": 4038606.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.46457603573799133, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8411226868629456, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4098498821258545, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2911316752433777, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.41335782408714294, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3930765688419342, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 16745.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.36560165882110596, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.818734347820282, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4274422228336334, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3876758813858032, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4265812635421753, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.40738946199417114, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8244.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4289180040359497, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7863841652870178, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.35651302337646484, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4896979033946991, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.352658748626709, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.32506248354911804, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 7911.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4886866509914398, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3876505.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8059791326522827, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.42747461795806885, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.43612581491470337, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4306240379810333, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.409907728433609, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16055.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4500155746936798, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7846867442131042, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5418316721916199, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.592883288860321, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.539057195186615, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5103098154067993, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1911.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4484563171863556, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8447701930999756, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3554707169532776, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2551225423812866, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.35385626554489136, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3318524658679962, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 14934.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3644033670425415, "metrics/hit_grouping/calorimeter/purity": 0.48189952969551086, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8854823708534241, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05436766892671585, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14307692646980286, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05576923117041588, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.020530853420495987, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 650.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04769230633974075, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8357650637626648, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14433316886425018, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11744584143161774, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14432504773139954, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1266724318265915, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1754.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.28848347067832947, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8202332854270935, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4236176908016205, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.38188615441322327, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.42428189516067505, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4025651514530182, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 30496.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.42585912346839905, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.681191086769104, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4548714756965637, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.39772284030914307, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4107086956501007, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6692.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6719964146614075, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.766798198223114, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.41278743743896484, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45909199118614197, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.425432950258255, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1783.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.506449818611145, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6204854249954224, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4591635763645172, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3250004053115845, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.34116965532302856, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 12697.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.747814416885376, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6519948244094849, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.45390138030052185, "metrics/hit_grouping/tracker/all/object_completeness": 0.3592788875102997, "metrics/hit_grouping/tracker/all/object_purity": 0.37024563550949097, "metrics/hit_grouping/tracker/all/object_samples": 21172.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7035235166549683, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.004581265617161989, "metrics/hit_grouping/tracker/background_recall": 0.025469407439231873, "metrics/hit_grouping/tracker/completeness": 0.3795558512210846, "metrics/hit_grouping/tracker/hit_samples": 327857.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5393227338790894, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6706512570381165, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4483826756477356, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4313144087791443, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.44052502512931824, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 10975.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6885648965835571, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.625908613204956, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46297353506088257, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3127296268939972, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3421589136123657, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5253.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7403388619422913, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6382968425750732, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4565129578113556, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2488284558057785, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2440771758556366, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 4944.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6976132392883301, "metrics/hit_grouping/tracker/owned_hit_samples": 310974.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.653861403465271, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4683680832386017, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3496304452419281, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3750554919242859, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11128.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7005751132965088, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6955265402793884, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.49334222078323364, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3962605893611908, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3889964818954468, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1502.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6344873309135437, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6419087052345276, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.42811986804008484, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3653455674648285, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.36068257689476013, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 8542.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.71950364112854, "metrics/hit_grouping/tracker/purity": 0.3616675138473511, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8858416080474854, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.016507385298609734, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04845811054110527, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0527677945792675, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2302.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.09165942668914795, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6646091341972351, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19232763350009918, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.18168485164642334, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1732483059167862, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1929.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5873509645462036, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.618782639503479, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.543120265007019, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.42173612117767334, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4358168840408325, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 16941.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7998937368392944, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.766706645488739, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.269697368144989, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.9340308904647827, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9390365481376648, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9849745631217957, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8315256834030151, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9544829726219177, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5891003608703613, "metrics/linear_probe/calorimeter/embedding/ari": 0.4551358222961426, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.428676962852478, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.4570462703704834, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5947951674461365, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.030591506510972977, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06399041414260864, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 63846.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 202077.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9722191095352173, "metrics/linear_probe/calorimeter/raw/ari": 0.48572733998298645, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.6529340147972107, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.7334586381912231, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5308047533035278, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.323795318603516, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8345605134963989, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9578210711479187, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6055532693862915, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.23718413710594177, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.10674429684877396, "metrics/linear_probe/tracker/oracle_graph/ari": 0.9052689075469971, "metrics/linear_probe/tracker/raw/ari": 0.13043981790542603, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6731104254722595, "metrics/particle/class_1/pid_accuracy": 0.9238072633743286, "metrics/particle/class_2/efficiency": 0.4324657917022705, "metrics/particle/class_2/pid_accuracy": 0.5203631520271301, "metrics/particle/class_3/efficiency": 0.5551026463508606, "metrics/particle/class_3/pid_accuracy": 0.9284637570381165, "metrics/particle/class_4/efficiency": 0.5620788931846619, "metrics/particle/class_4/pid_accuracy": 0.3544520437717438, "metrics/particle/class_5/efficiency": 0.7139999866485596, "metrics/particle/class_5/pid_accuracy": 0.40336135029792786, "metrics/particle/count/bias_mean": -7.559895992279053, "metrics/particle/count/mae": 8.361979484558105, "metrics/particle/count/prediction_mean": 62.49609375, "metrics/particle/count/target_mean": 70.05599212646484, "metrics/particle/event/energy_relative_abs_error": 0.09450183063745499, "metrics/particle/event/energy_response_mean": 0.9914297461509705, "metrics/particle/event/met_abs_error": 10.437966346740723, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0839414894580841, "metrics/particle/event/scalar_pt_response_mean": 1.0116090774536133, "metrics/particle/event/vector_pt_closure": 0.0789419412612915, "metrics/particle/matched/delta_eta_abs_mean": 0.02107330970466137, "metrics/particle/matched/delta_phi_abs_mean": 0.018796337768435478, "metrics/particle/matched/delta_r_mean": 0.03159632906317711, "metrics/particle/matched/energy_relative_abs_mean": 0.18124094605445862, "metrics/particle/matched/pid_accuracy": 0.89361572265625, "metrics/particle/matched/pt_relative_abs_mean": 0.17834508419036865, "metrics/particle/matching/duplicate_fraction": 0.10252723842859268, "metrics/particle/matching/efficiency": 0.6011746525764465, "metrics/particle/matching/f1": 0.635461688041687, "metrics/particle/matching/purity": 0.673896312713623, "metrics/particle/matching/target_coverage_dr0p05": 0.527489185333252, "metrics/particle/matching/target_coverage_dr0p10": 0.6802873611450195, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.39830493927001953, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6611713171005249, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.029752645641565323, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1764918863773346, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17244656383991241, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01346922293305397, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21346662938594818, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014409221708774567, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0336350202560425, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01757971942424774, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.532822608947754, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6593.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.383211612701416, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.609310507774353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8545552492141724, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6868976354599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2241492122411728, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.445321083068848, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025741709396243095, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5312434434890747, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0468289814889431, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9175803661346436, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9168.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6028712391853333, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8878329992294312, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.207236289978027, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.656264781951904, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6257779598236084, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7233861088752747, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.285096168518066, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.3706289529800415, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.632020950317383, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10936.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028215082362294197, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18464508652687073, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17872099578380585, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018595142289996147, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22017672657966614, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.036204744130373, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.953281044960022, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.003908645361661911, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.001158237457275, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1602.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.42767879366874695, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3626315593719482, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.5796806812286377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.416459321975708, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22338490188121796, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.505583763122559, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06369426846504211, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7114665508270264, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04993424192070961, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.160712718963623, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2198.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5302879810333252, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.512327194213867, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.540875434875488, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5014220476150513, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14497140049934387, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.02209015004336834, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.365494728088379, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6718589067459106, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.932825088500977, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3021.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.032515086233615875, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18230320513248444, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18037669360637665, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024403488263487816, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2213718295097351, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001686082105152309, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.38291436433792114, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.052432578057050705, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0227665901184082, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13048.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.44336289167404175, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.7353484630584717, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.1767959594726562, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.006117582321167, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.27457526326179504, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.632753372192383, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.009964625351130962, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8209319114685059, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06584774702787399, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.86824631690979, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20071.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.608667254447937, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.059746742248535, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.929502487182617, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.714568138122559, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7105778455734253, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.8752869367599487, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.992531776428223, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.1446928977966309, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.692914009094238, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21437.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.031333453953266144, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18067620694637299, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17779064178466797, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020571893081068993, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.218858003616333, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008238008245825768, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7032989263534546, "metrics/particle/momentum/all/accepted/pz_bias": -0.027044149115681648, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.125967264175415, "metrics/particle/momentum/all/accepted/samples": 21243.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.424724280834198, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.310983896255493, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.1109893321990967, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.9417133331298828, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.25629034638404846, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.440119743347168, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018322359770536423, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1602622270584106, "metrics/particle/momentum/all/assigned/pz_bias": -0.03187509626150131, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5187928676605225, "metrics/particle/momentum/all/assigned/samples": 31437.0, "metrics/particle/momentum/all/efficiency": 0.6001864671707153, "metrics/particle/momentum/all/event/energy_bias": 0.4352525770664215, "metrics/particle/momentum/all/event/p3_error_mean": 25.522863388061523, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.760564804077148, "metrics/particle/momentum/all/event/px_bias": 1.1913845539093018, "metrics/particle/momentum/all/event/py_bias": 2.6207633018493652, "metrics/particle/momentum/all/event/pz_abs_mean": 18.89873504638672, "metrics/particle/momentum/all/event/pz_bias": -4.187180519104004, "metrics/particle/momentum/all/event/pz_residual_rms": 29.016830444335938, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35394.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037846360355615616, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.18408198654651642, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18372955918312073, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015525472350418568, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2239074409008026, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007647843565791845, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.747917890548706, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03586776554584503, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.9204094409942627, "metrics/particle/momentum/class_1/accepted/samples": 11245.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.299493670463562, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9424416422843933, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.9904361963272095, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8047817945480347, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.20376013219356537, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.058539390563965, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014521880075335503, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0511447191238403, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01441732794046402, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.894059419631958, "metrics/particle/momentum/class_1/assigned/samples": 15425.0, "metrics/particle/momentum/class_1/efficiency": 0.6738778948783875, "metrics/particle/momentum/class_1/event/energy_bias": 13.156183242797852, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.177152633666992, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.133116722106934, "metrics/particle/momentum/class_1/event/px_bias": 0.3092558681964874, "metrics/particle/momentum/class_1/event/py_bias": 0.9866294264793396, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.738335609436035, "metrics/particle/momentum/class_1/event/pz_bias": -2.379577159881592, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.020645141601562, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16687.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0350637249648571, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21420106291770935, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2147458791732788, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06868389993906021, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25632378458976746, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04123711213469505, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0068352222442627, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.037745337933301926, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.677363872528076, "metrics/particle/momentum/class_2/accepted/samples": 1261.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5234711766242981, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6584875583648682, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8405475616455078, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4926605522632599, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28093159198760986, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.2516117095947266, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05337214842438698, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5037925243377686, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1839871108531952, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.699312686920166, "metrics/particle/momentum/class_2/assigned/samples": 2061.0, "metrics/particle/momentum/class_2/efficiency": 0.4235807955265045, "metrics/particle/momentum/class_2/event/energy_bias": -16.755550384521484, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.823793411254883, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.39586067199707, "metrics/particle/momentum/class_2/event/px_bias": -0.2828959822654724, "metrics/particle/momentum/class_2/event/py_bias": -0.1272885948419571, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.52025318145752, "metrics/particle/momentum/class_2/event/pz_bias": -1.506704330444336, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.379980087280273, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2977.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021979328244924545, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17110641300678253, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16424131393432617, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04142047464847565, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20526710152626038, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0030904551967978477, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.41196754574775696, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01997852697968483, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.6619740724563599, "metrics/particle/momentum/class_3/accepted/samples": 8413.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5541845560073853, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.103191375732422, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.5796005725860596, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.4573636054992676, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3133254051208496, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.329410552978516, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016471287235617638, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0544822216033936, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.035291556268930435, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.7015750408172607, "metrics/particle/momentum/class_3/assigned/samples": 13478.0, "metrics/particle/momentum/class_3/efficiency": 0.5532320737838745, "metrics/particle/momentum/class_3/event/energy_bias": 6.068549156188965, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.035555839538574, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.510438919067383, "metrics/particle/momentum/class_3/event/px_bias": 1.2813184261322021, "metrics/particle/momentum/class_3/event/py_bias": 1.0804953575134277, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.587577819824219, "metrics/particle/momentum/class_3/event/pz_bias": -0.6345522403717041, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.83622169494629, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15207.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.035267870873212814, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17003539204597473, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16793805360794067, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.021760594099760056, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21458885073661804, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.024038461968302727, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.3311638832092285, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17884120345115662, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.907071590423584, "metrics/particle/momentum/class_4/accepted/samples": 208.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5091831088066101, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.314784526824951, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.2597782611846924, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.100902795791626, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2721712589263916, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.962759017944336, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.030581040307879448, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.5747437477111816, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2565775513648987, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.945827007293701, "metrics/particle/momentum/class_4/assigned/samples": 327.0, "metrics/particle/momentum/class_4/efficiency": 0.5714285969734192, "metrics/particle/momentum/class_4/event/energy_bias": -0.5805944204330444, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.9528207778930664, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.4992241859436035, "metrics/particle/momentum/class_4/event/px_bias": 0.2811233699321747, "metrics/particle/momentum/class_4/event/py_bias": 0.19174566864967346, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.5887837409973145, "metrics/particle/momentum/class_4/event/pz_bias": 0.11493122577667236, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.678666591644287, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 364.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03078506328165531, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19922059774398804, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2006901651620865, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.034150395542383194, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24292582273483276, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0517241396009922, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2108335494995117, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.06302965432405472, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.108284950256348, "metrics/particle/momentum/class_5/accepted/samples": 116.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12117113173007965, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2900616526603699, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2844102680683136, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.001148533308878541, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15753424167633057, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3810267448425293, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06849315017461777, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5595109462738037, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05968491733074188, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.789126873016357, "metrics/particle/momentum/class_5/assigned/samples": 146.0, "metrics/particle/momentum/class_5/efficiency": 0.7295597195625305, "metrics/particle/momentum/class_5/event/energy_bias": -1.4533332586288452, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.2997097969055176, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.5077152252197266, "metrics/particle/momentum/class_5/event/px_bias": -0.39741721749305725, "metrics/particle/momentum/class_5/event/py_bias": 0.48918142914772034, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7239060401916504, "metrics/particle/momentum/class_5/event/pz_bias": 0.21872156858444214, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.459235668182373, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 159.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5545561909675598, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5576271414756775, "metrics/set_slot/angular_recall_all_dr0p10": 0.8598777055740356, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6926007866859436, "metrics/set_slot/soft_count_bias_mean": 0.19009315967559814}, "step_train_time": 5.5840630531311035, "step_valid_time": 27746.4807369709, "step_total_time": 27752.06480050087, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0336812130060848, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21841039388553796, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3027, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2775, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2713, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9167492566897919, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9113300492610837, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9140316205533597, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08866995073891626, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8962669309547406, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8909688013136289, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8936100131752306, "step/clic_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9167492566897919, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.018435862019369, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21267961145443426, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2262, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2081, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2052, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9199823165340407, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.925300133392619, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9226335624030149, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07469986660738105, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9071618037135278, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.912405513561583, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9097761028596764, "step/clic_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9199823165340407, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016298176267659, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2111625083516362, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1288, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1263, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1137, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1111, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8827639751552795, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9002375296912114, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8914151313210505, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0997624703087886, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8625776397515528, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8796516231195566, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.871030968247746, "step/clic_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8827639751552795} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/hyperparameters.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..a1459e39e344065372def5adb34c3f730247fb01 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 12786964, "dataset": "clic_hits", "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "heptv2", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "set", "binary_classification_focal_gamma": 2.0, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": true, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": {"num_slots": 256, "num_layers": 4, "num_heads": 8, "ffn_multiplier": 4.0, "dropout": 0.0, "query_init": "input-conditioned", "local_attention_radius": 0.4, "attention_mode": "global-flash", "num_sectors": 32, "sector_neighbors": 1, "tracker_query_fraction": 0.6, "proposal_mode": "grid-diverse", "proposal_grid_size": 0.1, "use_aggregate_anchors": true, "aggregate_grid_size": 0.1, "presence_threshold": 0.5, "no_object_weight": 1.0, "cardinality_loss_weight": 0.05, "auxiliary_loss_weight": 0.25, "momentum_loss": "mse", "momentum_huber_delta": 0.2, "hit_grouping": true, "hit_grouping_dim": 32, "hit_grouping_chunk_size": 1024, "hit_grouping_loss_weight": 0.1, "hit_grouping_background_weight": 0.1, "hit_grouping_detector_balanced": false, "hit_grouping_tracker_particle_balanced": false, "tracker_overlap_loss_weight": 0.0, "hit_grouping_sparse": false, "hit_grouping_topk": 4, "hit_grouping_temperature": 1.0, "tracker_momentum_readout": true, "tracker_momentum_max_log_correction": 0.4054651081081644, "tracker_geometry_loss_weight": 0.0, "tracker_curvature_loss_weight": 0.0, "tracker_loss_max_hits": 64, "tracker_group_readout": false, "tracker_fit_correction": false, "tracker_fit_max_log_correction": 0.4054651081081644, "tracker_group_magnetic_field": 2.0, "event_consistency_loss_weight": 0.0, "matcher": {"presence": 1.0, "pid": 1.0, "geometry": 2.0, "pt": 1.0, "energy": 0.0, "dr_scale": 0.1, "log_pt_scale": 0.6931471805599453, "log_energy_scale": 0.6931471805599453}}, "gnnlsh": null, "attention": null, "litept": null, "hept": null, "heptv2": {"activation_checkpointing": true, "embedding_dim": 256, "width": 256, "num_convs": 6, "num_heads": 16, "dropout_ff": 0.0, "activation": "elu", "pe_type": "learned", "block_size": 128, "n_hashes": 3, "num_regions": 140, "deterministic_hashing": true, "num_w_per_dist": 10, "mlp_ratio": 4.0}}, "conv_type": "heptv2", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0001}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 15, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth", "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"clic_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"clic_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["clic_edm_ttbar_hits", "clic_edm_ww_fullhad_hits", "clic_edm_qq_hits"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/model_kwargs.pkl b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d451a4832aff821c6e5486c8910d0e624a84770d --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88528787cbe2db8ee76475a6c6d3a94fb412233b068a26a15e912a35241265e1 +size 6763 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/particleflow_spec.yaml b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e0f8b552375d1a1eba1de114815c03c407678e5b Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..4e256eb12d61d6bbf2485bbe2b0260ce14d3e421 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25c4a3c83815d5bc4486d70be8fb5d503ede7a24 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..bf0e33800cff28bb0ef7609dcd4993b25039c6f0 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a08b1a5fc090d30761f147c547d56b5a94b279c2 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..f9ce74ee4117f43f50a8e19e14554eb2edd7f760 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7264e4a2b2eb7da8e214cc7d94a146446fd1bd14 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..eed1ab9973b944940215b34a5b1ffd9b3d24112f Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0b3a4487a1db14699af15b721c7fab99edef1cb2 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..1011cd8aa96c62f9ee69732afa479bca15a9140a Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..be8458d1d05dcfe2e26404574694831534bcc576 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..fee592b5b26bc481b47023b979c65a26aeaeb703 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_qq_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25c5df69872f4cf81c34a1fe7ae6fab3d7c848bd Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..359613d1f3c4c5fbebc933106e2c50dce0866470 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f5159bff53199d78c56c3b78bba89af3d9d472c9 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..d9e5abe8203e38c16487da4a03a01db4fe0a7458 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5881b11babda609f059fdcb94ed344661e2975d1 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..84f5d43c33e846750f71c9e437c145faddbea1d5 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..99b6ac3f3ed8817df3483b3c1cfd21fc1196a15c Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..0a20b80bf5f8473e9310c486fc5bd5dc92d251b5 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7ff14675f8a2c5e05c6add084d0992f4b0e8b3ac Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..240a190e9338a47aad3140b63bebb965e3b1c09e Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..79cad93f0adf8df7f395399b33879a6731d36fd1 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..f8cb58fe66c9884d2e8a03f2839436198ac95b1e Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ttbar_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fba722ddffd203a933fb2404eb963621ee2a1106 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..dde94132636d49386c55129d1ecc862f3f54b3b6 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_eta.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..920ff7a15b3b93cc4e115252031c278e831ae8da Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..6af064a8368837cee3bc874e029365495d999c60 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8d02c4b6ec5733e587220ea01359786c6776fca4 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d64ac28b3ddf65a4c17dc58a0d1182f3e8f8da4a Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_pt_log.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0c8a531f8562644725e83ddb4f5d5bbd93bdecca Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..c7e38e65e68e4ad3122f240d5661ed54f10e95d4 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6d0c617bd03f1c1052f9b5610a65fb54044e9604 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..55610fa08186bedd88e666120939df28d6b823a3 Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/jet_res_vstarget.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a58838c8f4d70e7a8ed1f446af002e06fc5c011f Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.pdf differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..5ff8d3b9e179bf1495bdda58b7e1569578361e4b Binary files /dev/null and b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/plots_step_50000/clic_edm_ww_fullhad_hits/num_elements.png differ diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793178800.nid006984.123108.0 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793178800.nid006984.123108.0 new file mode 100644 index 0000000000000000000000000000000000000000..ff580fa9c225716154b418dbfdcce99c090f24fc --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793178800.nid006984.123108.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9c4ca47c92c205a6d6a6855d0be1cc676d4b2c72b6dcb06265b3c65ab7c035 +size 27851551 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793278220.nid006205.64228.0 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793278220.nid006205.64228.0 new file mode 100644 index 0000000000000000000000000000000000000000..16cc84ac8e320dbff39332e46d052a5139919747 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/train/events.out.tfevents.1793278220.nid006205.64228.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2331584585894d63fbcbd2feb4037613dd54028ae5929dd4f843f037ff1dd1a3 +size 20146711 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793178800.nid006984.123108.1 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793178800.nid006984.123108.1 new file mode 100644 index 0000000000000000000000000000000000000000..d3ebd7ddb841e43e6b4e66fc63c04e0a1f8ec7b9 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793178800.nid006984.123108.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4813742d3858077937ced34ecb8d60f44478e60a1b1846d3dd28be3d4f0fea +size 1825802 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793278220.nid006205.64228.1 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793278220.nid006205.64228.1 new file mode 100644 index 0000000000000000000000000000000000000000..c3ef8a29df0e452bc35c29e12ef6950fc3e80e05 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/runs/valid/events.out.tfevents.1793278220.nid006205.64228.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8027f914aaae8f6a63b2cda85bfaf3d2e10326c974e6ae23a3a36c10015ef244 +size 1329514 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/scenario-manifest.json b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8255f9b234cf699a199ac01cb0bdfffe1deec68a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/scenario-manifest.json @@ -0,0 +1,1224 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239269-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239269-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "clic_hits_heptv2", + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "clic_hits", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "heptv2", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "set", + "binary_classification_focal_gamma": 2.0, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": true, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": { + "num_slots": 256, + "num_layers": 4, + "num_heads": 8, + "ffn_multiplier": 4.0, + "dropout": 0.0, + "query_init": "input-conditioned", + "local_attention_radius": 0.4, + "attention_mode": "global-flash", + "num_sectors": 32, + "sector_neighbors": 1, + "tracker_query_fraction": 0.6, + "proposal_mode": "grid-diverse", + "proposal_grid_size": 0.1, + "use_aggregate_anchors": true, + "aggregate_grid_size": 0.1, + "presence_threshold": 0.5, + "no_object_weight": 1.0, + "cardinality_loss_weight": 0.05, + "auxiliary_loss_weight": 0.25, + "momentum_loss": "mse", + "momentum_huber_delta": 0.2, + "hit_grouping": true, + "hit_grouping_dim": 32, + "hit_grouping_chunk_size": 1024, + "hit_grouping_loss_weight": 0.1, + "hit_grouping_background_weight": 0.1, + "hit_grouping_detector_balanced": false, + "hit_grouping_tracker_particle_balanced": false, + "tracker_overlap_loss_weight": 0.0, + "hit_grouping_sparse": false, + "hit_grouping_topk": 4, + "hit_grouping_temperature": 1.0, + "tracker_momentum_readout": true, + "tracker_momentum_max_log_correction": 0.4054651081081644, + "tracker_geometry_loss_weight": 0.0, + "tracker_curvature_loss_weight": 0.0, + "tracker_loss_max_hits": 64, + "tracker_group_readout": false, + "tracker_fit_correction": false, + "tracker_fit_max_log_correction": 0.4054651081081644, + "tracker_group_magnetic_field": 2.0, + "event_consistency_loss_weight": 0.0, + "matcher": { + "presence": 1.0, + "pid": 1.0, + "geometry": 2.0, + "pt": 1.0, + "energy": 0.0, + "dr_scale": 0.1, + "log_pt_scale": 0.6931471805599453, + "log_energy_scale": 0.6931471805599453 + } + }, + "gnnlsh": null, + "attention": null, + "litept": null, + "hept": null, + "heptv2": { + "activation_checkpointing": true, + "embedding_dim": 256, + "width": 256, + "num_convs": 6, + "num_heads": 16, + "dropout_ff": 0.0, + "activation": "elu", + "pe_type": "learned", + "block_size": 128, + "n_hashes": 3, + "num_regions": 140, + "deterministic_hashing": true, + "num_w_per_dist": 10, + "mlp_ratio": 4.0 + } + }, + "conv_type": "heptv2", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0001 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 15, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "clic_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "clic_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "clic_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "clic_edm_ttbar_hits", + "clic_edm_ww_fullhad_hits", + "clic_edm_qq_hits" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-hits-set-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "continuations": [ + { + "checkpoint": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth", + "step": 29000, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-hits-set-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--load", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "submitted_at": "2026-10-29T14:48:55.134225" + } + ] +} \ No newline at end of file diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train-config.yaml b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f94ab9be52dae558bf765f33fab0e7ea242a8e7 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train-config.yaml @@ -0,0 +1,327 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: heptv2 +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds +dataset: clic_hits +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- clic_edm_ttbar_hits +- clic_edm_ww_fullhad_hits +- clic_edm_qq_hits +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0001 +input_dim: 15 +load: /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: null + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: 2.0 + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: + activation: elu + activation_checkpointing: true + block_size: 128 + deterministic_hashing: true + dropout_ff: 0.0 + embedding_dim: 256 + mlp_ratio: 4.0 + n_hashes: 3 + num_convs: 6 + num_heads: 16 + num_regions: 140 + num_w_per_dist: 10 + pe_type: learned + width: 256 + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: true + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: set + pt_mode: direct-elemtype-split + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + dropout: 0.0 + event_consistency_loss_weight: 0.0 + ffn_multiplier: 4.0 + hit_grouping: true + hit_grouping_background_weight: 0.1 + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + hit_grouping_sparse: false + hit_grouping_temperature: 1.0 + hit_grouping_topk: 4 + hit_grouping_tracker_particle_balanced: false + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + momentum_huber_delta: 0.2 + momentum_loss: mse + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_curvature_loss_weight: 0.0 + tracker_fit_correction: false + tracker_fit_max_log_correction: 0.4054651081081644 + tracker_geometry_loss_weight: 0.0 + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_loss_max_hits: 64 + tracker_momentum_max_log_correction: 0.4054651081081644 + tracker_momentum_readout: true + tracker_overlap_loss_weight: 0.0 + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + sin_phi_mode: linear + task_queries: false + trainable: all + type: heptv2 +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + clic_edm_qq_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 +threads: 16 +train: true +train_dataset: + clic_hits: + physical: + batch_size: 1 + samples: + clic_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +val_freq: 1000 +valid_dataset: + clic_hits: + physical: + batch_size: 1 + samples: + clic_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + clic_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f66517def5a5e88f047bea640dde05acc309bf7a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log @@ -0,0 +1,20 @@ +rank0 [2026-10-28 11:12:24,690] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-28 11:12:24,888] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,889] INFO : Spawning DDP processes +rank0 [2026-10-29 14:49:18,727] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-29 14:49:18,933] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,933] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,933] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:49:18,934] INFO : Spawning DDP processes diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.0 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..a28f67a84926a24e340cb3a1c3c54edd5797594a --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.0 @@ -0,0 +1,15066 @@ +rank0 [2026-10-28 11:12:32,891] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:32,892] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank0 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-28 11:12:57,192] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank0 [2026-10-28 11:12:57,210] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:57,210] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:57,248] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:57,262] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:57,277] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:57,291] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:57,305] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:57,319] INFO : Backbone initialization took 0.11s +rank0 [2026-10-28 11:12:57,320] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:57,320] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:57,383] INFO : Output DNNs initialization took 0.06s +rank0 [2026-10-28 11:12:57,383] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:57,383] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:57,383] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:57,383] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:57,383] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:57,383] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:57,384] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:57,384] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:57,384] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:57,384] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:57,384] INFO : output_mode=set +rank0 [2026-10-28 11:12:57,384] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-28 11:12:57,385] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_pid parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_pt parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_eta parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-28 11:12:57,385] INFO : nn_energy parameters: 0 +rank0 [2026-10-28 11:12:57,386] INFO : Total MLPF parameters: 12786964 +rank0 [2026-10-28 11:12:57,386] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:57,387] INFO : Instantiated model +rank0 [2026-10-28 11:12:57,387] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:57,418] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:57,421] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:57,421] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-28 11:12:57,716] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-28 11:12:57,919] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank0 [2026-10-28 11:12:57,919] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-28 11:12:57,920] INFO : Trainable parameters: 12786964 +rank0 [2026-10-28 11:12:57,920] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:57,920] INFO : Total parameters: 12810232 +rank0 [2026-10-28 11:12:57,924] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-28 11:12:57,928] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645 +rank0 [2026-10-28 11:12:57,928] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645 +rank0 [2026-10-28 11:12:57,928] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:57,928] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:02,562] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,668] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,773] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,869] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,975] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,082] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,179] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,558] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,638] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,720] INFO : train_dataset: clic_edm_ttbar_hits, 90900 +rank0 [2026-10-28 11:13:03,720] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:03,806] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:03,910] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,010] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,135] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,252] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,341] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,459] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,560] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,661] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,762] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-28 11:13:04,762] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:04,860] INFO : train_dataset: clic_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,212] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,287] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-28 11:13:05,363] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,451] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-28 11:13:05,576] INFO : train_dataset: clic_edm_qq_hits, 89996 +rank0 [2026-10-28 11:13:05,680] INFO : train_dataset: clic_edm_qq_hits, 89998 +rank0 [2026-10-28 11:13:05,786] INFO : train_dataset: clic_edm_qq_hits, 89994 +rank0 [2026-10-28 11:13:05,895] INFO : train_dataset: clic_edm_qq_hits, 89995 +rank0 [2026-10-28 11:13:06,001] INFO : train_dataset: clic_edm_qq_hits, 90099 +rank0 [2026-10-28 11:13:06,001] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:13:06,001] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:13:06,001] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:13:06,001] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,028] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,055] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,082] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,109] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,135] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,163] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,199] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,235] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,262] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,287] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,287] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,313] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,339] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,364] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,389] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,415] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,440] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,466] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,493] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,524] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,552] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,552] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,576] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,602] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,627] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,652] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,678] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,703] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,728] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,754] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,783] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,808] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,809] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:13:06,809] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:13:20,622] INFO : Got dataloaders +rank0 [2026-10-28 11:13:20,622] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:13:20,622] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:13:20,622] INFO : Starting training from step 1 to 50000 on rank 0 of 8 +rank0 [2026-10-28 11:13:20,628] INFO : Creating train_iterator +rank0 [2026-10-28 11:13:20,629] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:13:35,190] INFO : Created train_iterator +rank0 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank0 [2026-10-28 11:18:17,225] INFO : Step 100/50000 rank0 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7475 | Classification_binary: 0.4963 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7247 | Regression_eta: 0.0129 | Regression_pt: 0.7050 | Regression_sin_phi: 0.0073 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 4.3312s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank0 [2026-10-28 11:18:17,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 60.0°C 202.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 55.0°C 202.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +6 10 0x7408, 62036 52.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:22:48,718] INFO : Step 200/50000 rank0 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3633 | Classification_binary: 0.5452 | Regression_cos_phi: 0.3621 | Regression_energy: 0.3890 | Regression_eta: 0.3150 | Regression_pt: 0.3916 | Regression_sin_phi: 0.3572 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 5.1915s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank0 [2026-10-28 11:22:48,954] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:27:18,516] INFO : Step 300/50000 rank0 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3173 | Classification_binary: 0.4642 | Regression_cos_phi: 0.3339 | Regression_energy: 0.3055 | Regression_eta: 0.2839 | Regression_pt: 0.3096 | Regression_sin_phi: 0.3073 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 4.9917s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank0 [2026-10-28 11:27:18,758] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 47.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:31:49,522] INFO : Step 400/50000 rank0 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2801 | Classification_binary: 0.4402 | Regression_cos_phi: 0.2986 | Regression_energy: 0.3244 | Regression_eta: 0.2689 | Regression_pt: 0.3121 | Regression_sin_phi: 0.3380 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.8728s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank0 [2026-10-28 11:31:49,759] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:36:20,814] INFO : Step 500/50000 rank0 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2598 | Classification_binary: 0.5185 | Regression_cos_phi: 0.3258 | Regression_energy: 0.3054 | Regression_eta: 0.2444 | Regression_pt: 0.3187 | Regression_sin_phi: 0.3024 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 5.1180s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank0 [2026-10-28 11:36:21,056] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:40:55,455] INFO : Step 600/50000 rank0 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2161 | Classification_binary: 0.4656 | Regression_cos_phi: 0.2374 | Regression_energy: 0.2796 | Regression_eta: 0.2463 | Regression_pt: 0.2919 | Regression_sin_phi: 0.2186 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 5.1893s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank0 [2026-10-28 11:40:55,697] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:45:33,232] INFO : Step 700/50000 rank0 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2320 | Classification_binary: 0.3639 | Regression_cos_phi: 0.2736 | Regression_energy: 0.2645 | Regression_eta: 0.2041 | Regression_pt: 0.2715 | Regression_sin_phi: 0.2557 | LR: 1.00e-03 | DataLoad Time: 0.0179s | Model Forward Time: 4.3982s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank0 [2026-10-28 11:45:33,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:50:15,932] INFO : Step 800/50000 rank0 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2522 | Classification_binary: 0.4767 | Regression_cos_phi: 0.2868 | Regression_energy: 0.2682 | Regression_eta: 0.2324 | Regression_pt: 0.2808 | Regression_sin_phi: 0.2494 | LR: 9.99e-04 | DataLoad Time: 0.0226s | Model Forward Time: 5.9941s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank0 [2026-10-28 11:50:16,172] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:54:59,195] INFO : Step 900/50000 rank0 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2255 | Classification_binary: 0.4539 | Regression_cos_phi: 0.2268 | Regression_energy: 0.3010 | Regression_eta: 0.1561 | Regression_pt: 0.3162 | Regression_sin_phi: 0.1872 | LR: 9.99e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1131s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank0 [2026-10-28 11:54:59,433] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:59:56,050] INFO : Step 1000/50000 rank0 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1773 | Classification_binary: 0.4271 | Regression_cos_phi: 0.2116 | Regression_energy: 0.2351 | Regression_eta: 0.1938 | Regression_pt: 0.2453 | Regression_sin_phi: 0.2754 | LR: 9.99e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.5502s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank0 [2026-10-28 11:59:56,284] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 47.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:59:56,352] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:59:56,632] INFO : Running validation on rank0 +rank0 [2026-10-28 11:59:56,632] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:59:56,632] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION | Step=1000/50000 | Train Loss=2.7920 | Valid Loss=3.0167 | Train Loss Unweighted=1.7295 | Valid Loss Unweighted=1.7937 | Stale=0 | ETA=2326.4m +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 12:00:49,302] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari=0.385563 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari_threshold=0.675 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.276013 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_competing=0.272781 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.515477 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.100165 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0153277 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 12:00:49,303] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/seconds_per_rank=22.3386 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:00:49,304] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari=0.146393 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari_threshold=0.83125 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding_minus_raw/ari=0.0159537 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 12:00:49,305] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 12:00:49,310] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:00:49,569] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 12:00:49,590] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 12:01:16,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:01:17,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:01:19,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:01:20,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:01:30,262] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:30,838] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:01:30,841] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:01:47,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:01:48,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:01:49,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:01:50,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:02:00,453] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 12:02:00,464] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:02:01,008] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 12:02:01,011] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 12:02:16,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:02:17,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:02:18,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:02:19,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:02:29,336] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 12:02:29,352] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:02:32,540] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:02:33,864] INFO : Plotted number of elements +rank0 [2026-10-28 12:02:35,821] INFO : Plotted jets +rank0 [2026-10-28 12:02:36,955] INFO : Plotted jet ratio +rank0 [2026-10-28 12:02:37,038] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:02:40,290] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:02:41,271] INFO : Plotted number of elements +rank0 [2026-10-28 12:02:42,775] INFO : Plotted jets +rank0 [2026-10-28 12:02:43,947] INFO : Plotted jet ratio +rank0 [2026-10-28 12:02:43,997] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits +rank0 [2026-10-28 12:02:46,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits +rank0 [2026-10-28 12:02:47,281] INFO : Plotted number of elements +rank0 [2026-10-28 12:02:49,260] INFO : Plotted jets +rank0 [2026-10-28 12:02:50,471] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:34,255] INFO : Step 1100/50000 rank0 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2111 | Classification_binary: 0.4308 | Regression_cos_phi: 0.2448 | Regression_energy: 0.2740 | Regression_eta: 0.1743 | Regression_pt: 0.2835 | Regression_sin_phi: 0.1960 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5165s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank0 [2026-10-28 12:07:34,493] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:12:21,019] INFO : Step 1200/50000 rank0 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2016 | Classification_binary: 0.4840 | Regression_cos_phi: 0.2626 | Regression_energy: 0.2678 | Regression_eta: 0.2026 | Regression_pt: 0.2776 | Regression_sin_phi: 0.2939 | LR: 9.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.1940s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank0 [2026-10-28 12:12:21,255] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:17:08,584] INFO : Step 1300/50000 rank0 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1857 | Classification_binary: 0.4014 | Regression_cos_phi: 0.1961 | Regression_energy: 0.2140 | Regression_eta: 0.1727 | Regression_pt: 0.2248 | Regression_sin_phi: 0.2517 | LR: 9.98e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.9923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank0 [2026-10-28 12:17:08,824] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:21:56,052] INFO : Step 1400/50000 rank0 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2014 | Classification_binary: 0.3637 | Regression_cos_phi: 0.2564 | Regression_energy: 0.2279 | Regression_eta: 0.1792 | Regression_pt: 0.2355 | Regression_sin_phi: 0.2343 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2950s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank0 [2026-10-28 12:21:56,289] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:26:34,258] INFO : Step 1500/50000 rank0 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2204 | Classification_binary: 0.4590 | Regression_cos_phi: 0.2282 | Regression_energy: 0.2424 | Regression_eta: 0.1785 | Regression_pt: 0.2571 | Regression_sin_phi: 0.1821 | LR: 9.98e-04 | DataLoad Time: 0.0141s | Model Forward Time: 5.6728s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank0 [2026-10-28 12:26:34,497] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 55.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:31:14,598] INFO : Step 1600/50000 rank0 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1878 | Classification_binary: 0.4350 | Regression_cos_phi: 0.2500 | Regression_energy: 0.2365 | Regression_eta: 0.2305 | Regression_pt: 0.2498 | Regression_sin_phi: 0.2051 | LR: 9.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.4656s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank0 [2026-10-28 12:31:14,837] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:36:04,867] INFO : Step 1700/50000 rank0 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1243 | Classification_binary: 0.3112 | Regression_cos_phi: 0.2101 | Regression_energy: 0.2131 | Regression_eta: 0.1426 | Regression_pt: 0.2199 | Regression_sin_phi: 0.1789 | LR: 9.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4448s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank0 [2026-10-28 12:36:05,103] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:40:54,886] INFO : Step 1800/50000 rank0 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2137 | Classification_binary: 0.4100 | Regression_cos_phi: 0.2482 | Regression_energy: 0.2398 | Regression_eta: 0.1837 | Regression_pt: 0.2458 | Regression_sin_phi: 0.1926 | LR: 9.97e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3129s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank0 [2026-10-28 12:40:55,123] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:45:38,264] INFO : Step 1900/50000 rank0 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1738 | Classification_binary: 0.4474 | Regression_cos_phi: 0.2029 | Regression_energy: 0.2167 | Regression_eta: 0.1822 | Regression_pt: 0.2249 | Regression_sin_phi: 0.2223 | LR: 9.97e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2543s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank0 [2026-10-28 12:45:38,500] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 54.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:50:23,444] INFO : Step 2000/50000 rank0 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1539 | Classification_binary: 0.3595 | Regression_cos_phi: 0.1681 | Regression_energy: 0.2097 | Regression_eta: 0.1589 | Regression_pt: 0.2196 | Regression_sin_phi: 0.1587 | LR: 9.96e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.8374s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank0 [2026-10-28 12:50:23,686] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 9% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 5% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 70% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:50:23,712] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 12:50:24,309] INFO : Running validation on rank0 +rank0 [2026-10-28 12:50:24,309] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:50:24,309] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION | Step=2000/50000 | Train Loss=2.4875 | Valid Loss=2.6024 | Train Loss Unweighted=1.4740 | Valid Loss Unweighted=1.5268 | Stale=0 | ETA=2351.9m +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 12:51:20,355] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari=0.424092 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.327521 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_competing=0.332558 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.546171 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0616355 +rank0 [2026-10-28 12:51:20,356] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0153661 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 12:51:20,357] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/seconds_per_rank=21.967 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari=0.175126 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding_minus_raw/ari=0.044686 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 12:51:20,358] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 12:51:20,359] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 12:51:20,363] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:51:20,710] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 12:51:20,727] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 12:51:47,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:51:48,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:51:50,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:51:51,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:52:01,698] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 12:52:01,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:02,240] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:52:02,244] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:52:18,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:52:19,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:52:20,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:52:21,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:52:31,396] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 12:52:31,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:32,091] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 12:52:32,094] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 12:52:48,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:52:49,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:52:50,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:52:51,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:53:01,622] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:53:01,639] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:53:04,909] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits +rank0 [2026-10-28 12:53:05,939] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:07,481] INFO : Plotted jets +rank0 [2026-10-28 12:53:08,613] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:08,686] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:53:11,972] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 12:53:12,920] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:14,416] INFO : Plotted jets +rank0 [2026-10-28 12:53:16,007] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:16,062] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits +rank0 [2026-10-28 12:53:18,478] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits +rank0 [2026-10-28 12:53:19,342] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:20,891] INFO : Plotted jets +rank0 [2026-10-28 12:53:22,066] INFO : Plotted jet ratio +rank0 [2026-10-28 12:58:22,589] INFO : Step 2100/50000 rank0 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1710 | Classification_binary: 0.3253 | Regression_cos_phi: 0.2279 | Regression_energy: 0.2103 | Regression_eta: 0.1586 | Regression_pt: 0.2226 | Regression_sin_phi: 0.2197 | LR: 9.96e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2951s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank0 [2026-10-28 12:58:22,825] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:03:25,794] INFO : Step 2200/50000 rank0 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2021 | Classification_binary: 0.4327 | Regression_cos_phi: 0.2665 | Regression_energy: 0.2164 | Regression_eta: 0.1870 | Regression_pt: 0.2250 | Regression_sin_phi: 0.2256 | LR: 9.96e-04 | DataLoad Time: 0.0182s | Model Forward Time: 7.1819s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank0 [2026-10-28 13:03:26,031] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 5% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 12% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 98% +4 8 0x7408, 13395 53.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 96% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 96% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:08:12,907] INFO : Step 2300/50000 rank0 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2010 | Classification_binary: 0.4077 | Regression_cos_phi: 0.2984 | Regression_energy: 0.2505 | Regression_eta: 0.2443 | Regression_pt: 0.2637 | Regression_sin_phi: 0.2592 | LR: 9.95e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.6910s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank0 [2026-10-28 13:08:13,144] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:13:01,307] INFO : Step 2400/50000 rank0 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1453 | Classification_binary: 0.3410 | Regression_cos_phi: 0.2069 | Regression_energy: 0.2435 | Regression_eta: 0.1360 | Regression_pt: 0.2531 | Regression_sin_phi: 0.2093 | LR: 9.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8713s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank0 [2026-10-28 13:13:01,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:18:07,768] INFO : Step 2500/50000 rank0 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1725 | Classification_binary: 0.4117 | Regression_cos_phi: 0.2700 | Regression_energy: 0.1982 | Regression_eta: 0.1889 | Regression_pt: 0.2080 | Regression_sin_phi: 0.2275 | LR: 9.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.8068s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank0 [2026-10-28 13:18:08,018] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 12% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 1% +2 6 0x7408, 15961 47.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 94% +4 8 0x7408, 13395 54.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +6 10 0x7408, 62036 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 98% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:22:58,333] INFO : Step 2600/50000 rank0 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1551 | Classification_binary: 0.3812 | Regression_cos_phi: 0.1686 | Regression_energy: 0.2716 | Regression_eta: 0.1468 | Regression_pt: 0.2860 | Regression_sin_phi: 0.2092 | LR: 9.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9738s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank0 [2026-10-28 13:22:58,571] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:27:40,611] INFO : Step 2700/50000 rank0 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1736 | Classification_binary: 0.3082 | Regression_cos_phi: 0.1520 | Regression_energy: 0.1915 | Regression_eta: 0.1176 | Regression_pt: 0.1925 | Regression_sin_phi: 0.1752 | LR: 9.94e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.3244s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank0 [2026-10-28 13:27:40,852] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 5% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 98% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 1% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 5% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:32:28,287] INFO : Step 2800/50000 rank0 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1626 | Classification_binary: 0.3736 | Regression_cos_phi: 0.1852 | Regression_energy: 0.2126 | Regression_eta: 0.1379 | Regression_pt: 0.2200 | Regression_sin_phi: 0.2011 | LR: 9.93e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9638s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank0 [2026-10-28 13:32:28,524] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 185.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:37:10,143] INFO : Step 2900/50000 rank0 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1560 | Classification_binary: 0.4029 | Regression_cos_phi: 0.2491 | Regression_energy: 0.2086 | Regression_eta: 0.1858 | Regression_pt: 0.2180 | Regression_sin_phi: 0.2380 | LR: 9.93e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2221s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank0 [2026-10-28 13:37:10,382] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:42:04,350] INFO : Step 3000/50000 rank0 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1261 | Classification_binary: 0.3478 | Regression_cos_phi: 0.2132 | Regression_energy: 0.1909 | Regression_eta: 0.1405 | Regression_pt: 0.1963 | Regression_sin_phi: 0.1854 | LR: 9.92e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.2587s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank0 [2026-10-28 13:42:04,589] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:42:04,693] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 13:42:04,981] INFO : Running validation on rank0 +rank0 [2026-10-28 13:42:04,982] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:42:04,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:42:58,447] INFO : VALIDATION | Step=3000/50000 | Train Loss=2.4249 | Valid Loss=2.4455 | Train Loss Unweighted=1.3719 | Valid Loss Unweighted=1.4447 | Stale=0 | ETA=2344.2m +rank0 [2026-10-28 13:42:58,447] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:42:58,447] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 13:42:58,448] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari=0.449449 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.348158 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_competing=0.358919 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.543516 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0362785 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0127115 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:42:58,449] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/seconds_per_rank=22.0687 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 13:42:58,450] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari=0.171211 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding_minus_raw/ari=0.0407709 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 13:42:58,451] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:42:58,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:42:58,711] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:42:58,723] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 13:43:25,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:43:26,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:43:27,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:43:28,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:43:38,814] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:43:39,402] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:43:39,404] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:43:55,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:43:56,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:43:57,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:43:58,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:44:08,550] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:09,101] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 13:44:09,105] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 13:44:24,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:44:25,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:44:26,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:44:27,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:44:37,279] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 13:44:37,315] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:44:40,616] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits +rank0 [2026-10-28 13:44:41,683] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:43,600] INFO : Plotted jets +rank0 [2026-10-28 13:44:44,726] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:44,796] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:44:48,135] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 13:44:49,118] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:50,628] INFO : Plotted jets +rank0 [2026-10-28 13:44:51,810] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:51,863] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits +rank0 [2026-10-28 13:44:54,338] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits +rank0 [2026-10-28 13:44:55,210] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:56,710] INFO : Plotted jets +rank0 [2026-10-28 13:44:57,861] INFO : Plotted jet ratio +rank0 [2026-10-28 13:49:30,514] INFO : Step 3100/50000 rank0 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1467 | Classification_binary: 0.3217 | Regression_cos_phi: 0.1929 | Regression_energy: 0.1947 | Regression_eta: 0.1543 | Regression_pt: 0.2016 | Regression_sin_phi: 0.2822 | LR: 9.91e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.5635s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank0 [2026-10-28 13:49:30,752] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:54:23,267] INFO : Step 3200/50000 rank0 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1786 | Classification_binary: 0.3984 | Regression_cos_phi: 0.2237 | Regression_energy: 0.2104 | Regression_eta: 0.1546 | Regression_pt: 0.2208 | Regression_sin_phi: 0.2488 | LR: 9.91e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.6641s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank0 [2026-10-28 13:54:23,507] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:58:57,742] INFO : Step 3300/50000 rank0 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1673 | Classification_binary: 0.3666 | Regression_cos_phi: 0.2351 | Regression_energy: 0.1950 | Regression_eta: 0.1670 | Regression_pt: 0.2078 | Regression_sin_phi: 0.2019 | LR: 9.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3693s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank0 [2026-10-28 13:58:57,983] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:03:33,979] INFO : Step 3400/50000 rank0 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1108 | Classification_binary: 0.2551 | Regression_cos_phi: 0.1340 | Regression_energy: 0.2035 | Regression_eta: 0.1038 | Regression_pt: 0.2112 | Regression_sin_phi: 0.1408 | LR: 9.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5773s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank0 [2026-10-28 14:03:34,229] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 188.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:08:17,034] INFO : Step 3500/50000 rank0 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1661 | Classification_binary: 0.4029 | Regression_cos_phi: 0.2373 | Regression_energy: 0.2044 | Regression_eta: 0.1615 | Regression_pt: 0.2154 | Regression_sin_phi: 0.2297 | LR: 9.89e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3825s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank0 [2026-10-28 14:08:17,271] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:13:02,758] INFO : Step 3600/50000 rank0 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1587 | Classification_binary: 0.4196 | Regression_cos_phi: 0.2772 | Regression_energy: 0.2215 | Regression_eta: 0.1627 | Regression_pt: 0.2336 | Regression_sin_phi: 0.2319 | LR: 9.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4743s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank0 [2026-10-28 14:13:02,994] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 20% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:17:43,214] INFO : Step 3700/50000 rank0 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1402 | Classification_binary: 0.3255 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1847 | Regression_eta: 0.1471 | Regression_pt: 0.1933 | Regression_sin_phi: 0.2363 | LR: 9.88e-04 | DataLoad Time: 0.0087s | Model Forward Time: 6.0858s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank0 [2026-10-28 14:17:43,454] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 14% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 14% +2 6 0x7408, 15961 47.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +4 8 0x7408, 13395 54.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 95% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:22:45,005] INFO : Step 3800/50000 rank0 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1140 | Classification_binary: 0.3158 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1908 | Regression_eta: 0.1097 | Regression_pt: 0.1988 | Regression_sin_phi: 0.1460 | LR: 9.87e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.8421s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank0 [2026-10-28 14:22:45,242] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:27:34,543] INFO : Step 3900/50000 rank0 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1762 | Classification_binary: 0.4104 | Regression_cos_phi: 0.2162 | Regression_energy: 0.1935 | Regression_eta: 0.1350 | Regression_pt: 0.2010 | Regression_sin_phi: 0.2056 | LR: 9.87e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.6499s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank0 [2026-10-28 14:27:34,798] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 76% +2 6 0x7408, 15961 47.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 52% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 53.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 3% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 37% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:32:39,338] INFO : Step 4000/50000 rank0 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1414 | Classification_binary: 0.3326 | Regression_cos_phi: 0.1866 | Regression_energy: 0.2073 | Regression_eta: 0.1248 | Regression_pt: 0.2175 | Regression_sin_phi: 0.1554 | LR: 9.86e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2775s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank0 [2026-10-28 14:32:39,574] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:32:39,629] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 14:32:39,916] INFO : Running validation on rank0 +rank0 [2026-10-28 14:32:39,917] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:32:39,917] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:33:34,711] INFO : VALIDATION | Step=4000/50000 | Train Loss=2.2797 | Valid Loss=2.3380 | Train Loss Unweighted=1.3353 | Valid Loss Unweighted=1.3412 | Stale=0 | ETA=2302.7m +rank0 [2026-10-28 14:33:34,711] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 14:33:34,712] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari=0.460954 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.353524 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_competing=0.363625 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.541119 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0247729 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.010314 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 14:33:34,713] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/seconds_per_rank=22.094 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari=0.189845 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 14:33:34,714] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding_minus_raw/ari=0.0594056 +rank0 [2026-10-28 14:33:34,715] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 14:33:34,715] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 14:33:34,715] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 14:33:34,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:33:34,971] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 14:33:34,975] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 14:34:05,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 14:34:11,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 14:34:12,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 14:34:13,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 14:34:23,678] INFO : Time taken to make predictions on device 0 is: 0.39 min +rank0 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:24,284] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 14:34:24,288] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:34:40,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 14:34:42,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 14:34:44,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 14:34:45,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 14:34:55,333] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:56,345] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 14:34:56,349] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 14:35:11,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 14:35:12,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 14:35:13,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 14:35:14,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 14:35:24,975] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 14:35:24,994] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits +rank0 [2026-10-28 14:35:28,288] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits +rank0 [2026-10-28 14:35:29,401] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:30,920] INFO : Plotted jets +rank0 [2026-10-28 14:35:32,088] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:32,224] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:35:35,594] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 14:35:36,624] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:38,129] INFO : Plotted jets +rank0 [2026-10-28 14:35:39,265] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:39,320] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits +rank0 [2026-10-28 14:35:41,825] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits +rank0 [2026-10-28 14:35:43,182] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:44,697] INFO : Plotted jets +rank0 [2026-10-28 14:35:45,875] INFO : Plotted jet ratio +rank0 [2026-10-28 14:40:25,962] INFO : Step 4100/50000 rank0 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1525 | Classification_binary: 0.3252 | Regression_cos_phi: 0.1996 | Regression_energy: 0.2114 | Regression_eta: 0.1565 | Regression_pt: 0.2164 | Regression_sin_phi: 0.1856 | LR: 9.85e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5338s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank0 [2026-10-28 14:40:26,210] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 27% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:45:08,716] INFO : Step 4200/50000 rank0 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1671 | Classification_binary: 0.3629 | Regression_cos_phi: 0.2051 | Regression_energy: 0.1768 | Regression_eta: 0.1614 | Regression_pt: 0.1822 | Regression_sin_phi: 0.1970 | LR: 9.84e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.2192s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank0 [2026-10-28 14:45:08,953] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 1% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 86% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:49:56,224] INFO : Step 4300/50000 rank0 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.3400 | Regression_cos_phi: 0.1973 | Regression_energy: 0.1600 | Regression_eta: 0.1191 | Regression_pt: 0.1638 | Regression_sin_phi: 0.1594 | LR: 9.84e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1708s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank0 [2026-10-28 14:49:56,461] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:54:32,636] INFO : Step 4400/50000 rank0 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0914 | Classification_binary: 0.2877 | Regression_cos_phi: 0.1702 | Regression_energy: 0.1950 | Regression_eta: 0.1074 | Regression_pt: 0.2040 | Regression_sin_phi: 0.1126 | LR: 9.83e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.8660s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank0 [2026-10-28 14:54:32,871] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:59:29,439] INFO : Step 4500/50000 rank0 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1690 | Classification_binary: 0.3319 | Regression_cos_phi: 0.2089 | Regression_energy: 0.2001 | Regression_eta: 0.1580 | Regression_pt: 0.2059 | Regression_sin_phi: 0.2196 | LR: 9.82e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9341s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank0 [2026-10-28 14:59:29,689] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:04:25,627] INFO : Step 4600/50000 rank0 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1571 | Classification_binary: 0.4069 | Regression_cos_phi: 0.1780 | Regression_energy: 0.1943 | Regression_eta: 0.1328 | Regression_pt: 0.2039 | Regression_sin_phi: 0.1723 | LR: 9.81e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5154s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank0 [2026-10-28 15:04:25,868] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:09:19,321] INFO : Step 4700/50000 rank0 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.3168 | Regression_cos_phi: 0.1750 | Regression_energy: 0.1873 | Regression_eta: 0.1133 | Regression_pt: 0.1901 | Regression_sin_phi: 0.1299 | LR: 9.81e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.4653s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank0 [2026-10-28 15:09:19,558] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:14:31,472] INFO : Step 4800/50000 rank0 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1537 | Classification_binary: 0.3360 | Regression_cos_phi: 0.2110 | Regression_energy: 0.2126 | Regression_eta: 0.1660 | Regression_pt: 0.2222 | Regression_sin_phi: 0.2049 | LR: 9.80e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.1581s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank0 [2026-10-28 15:14:31,712] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 47.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:19:28,316] INFO : Step 4900/50000 rank0 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1351 | Classification_binary: 0.3867 | Regression_cos_phi: 0.1936 | Regression_energy: 0.1704 | Regression_eta: 0.1353 | Regression_pt: 0.1764 | Regression_sin_phi: 0.1773 | LR: 9.79e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2790s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank0 [2026-10-28 15:19:28,556] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:24:24,742] INFO : Step 5000/50000 rank0 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.3073 | Regression_cos_phi: 0.1354 | Regression_energy: 0.2036 | Regression_eta: 0.0902 | Regression_pt: 0.2079 | Regression_sin_phi: 0.1251 | LR: 9.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2265s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank0 [2026-10-28 15:24:24,980] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:24:25,005] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 15:24:25,392] INFO : Running validation on rank0 +rank0 [2026-10-28 15:24:25,392] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:24:25,392] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION | Step=5000/50000 | Train Loss=2.2286 | Valid Loss=2.2287 | Train Loss Unweighted=1.2644 | Valid Loss Unweighted=1.2595 | Stale=0 | ETA=2267.9m +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 15:25:19,702] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari=0.479604 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.373413 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_competing=0.389378 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.544729 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00612327 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0139244 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 15:25:19,703] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/seconds_per_rank=21.7005 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 15:25:19,704] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari=0.198871 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding_minus_raw/ari=0.0684308 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 15:25:19,705] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 15:25:19,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:19,969] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 15:25:19,974] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 15:25:45,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 15:25:46,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 15:25:47,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 15:25:48,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 15:25:58,730] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:59,251] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 15:25:59,255] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:26:15,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 15:26:16,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 15:26:17,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 15:26:18,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 15:26:28,874] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:26:28,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:26:29,404] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 15:26:29,412] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 15:26:45,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 15:26:46,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 15:26:46,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 15:26:48,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 15:26:58,018] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 15:26:58,034] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits +rank0 [2026-10-28 15:27:01,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits +rank0 [2026-10-28 15:27:02,323] INFO : Plotted number of elements +rank0 [2026-10-28 15:27:03,865] INFO : Plotted jets +rank0 [2026-10-28 15:27:05,033] INFO : Plotted jet ratio +rank0 [2026-10-28 15:27:05,085] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:27:08,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 15:27:09,344] INFO : Plotted number of elements +rank0 [2026-10-28 15:27:10,839] INFO : Plotted jets +rank0 [2026-10-28 15:27:12,390] INFO : Plotted jet ratio +rank0 [2026-10-28 15:27:12,446] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits +rank0 [2026-10-28 15:27:14,782] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits +rank0 [2026-10-28 15:27:15,631] INFO : Plotted number of elements +rank0 [2026-10-28 15:27:17,123] INFO : Plotted jets +rank0 [2026-10-28 15:27:18,295] INFO : Plotted jet ratio +rank0 [2026-10-28 15:32:02,560] INFO : Step 5100/50000 rank0 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.2899 | Regression_cos_phi: 0.1526 | Regression_energy: 0.1867 | Regression_eta: 0.1396 | Regression_pt: 0.1943 | Regression_sin_phi: 0.1362 | LR: 9.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 7.2698s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank0 [2026-10-28 15:32:02,800] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 28% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 53.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:36:54,209] INFO : Step 5200/50000 rank0 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1653 | Classification_binary: 0.3619 | Regression_cos_phi: 0.1599 | Regression_energy: 0.1817 | Regression_eta: 0.1423 | Regression_pt: 0.1913 | Regression_sin_phi: 0.1635 | LR: 9.76e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.3333s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank0 [2026-10-28 15:36:54,445] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 2% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 11% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 11% +6 10 0x7408, 62036 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 88% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:41:37,274] INFO : Step 5300/50000 rank0 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1327 | Classification_binary: 0.3553 | Regression_cos_phi: 0.1830 | Regression_energy: 0.1904 | Regression_eta: 0.1475 | Regression_pt: 0.1959 | Regression_sin_phi: 0.2161 | LR: 9.75e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6714s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank0 [2026-10-28 15:41:37,513] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:46:34,223] INFO : Step 5400/50000 rank0 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1366 | Classification_binary: 0.3048 | Regression_cos_phi: 0.1384 | Regression_energy: 0.1584 | Regression_eta: 0.1169 | Regression_pt: 0.1633 | Regression_sin_phi: 0.1848 | LR: 9.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.2503s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank0 [2026-10-28 15:46:34,461] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 1% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 86% +4 8 0x7408, 13395 55.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:51:27,743] INFO : Step 5500/50000 rank0 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.3056 | Regression_cos_phi: 0.1442 | Regression_energy: 0.1963 | Regression_eta: 0.1234 | Regression_pt: 0.2027 | Regression_sin_phi: 0.1453 | LR: 9.73e-04 | DataLoad Time: 0.0195s | Model Forward Time: 7.4016s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank0 [2026-10-28 15:51:27,980] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:56:26,426] INFO : Step 5600/50000 rank0 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1844 | Classification_binary: 0.4027 | Regression_cos_phi: 0.2404 | Regression_energy: 0.2060 | Regression_eta: 0.2070 | Regression_pt: 0.2186 | Regression_sin_phi: 0.2782 | LR: 9.72e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.0340s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank0 [2026-10-28 15:56:26,666] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:01:12,993] INFO : Step 5700/50000 rank0 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1099 | Classification_binary: 0.3002 | Regression_cos_phi: 0.1351 | Regression_energy: 0.1637 | Regression_eta: 0.1152 | Regression_pt: 0.1713 | Regression_sin_phi: 0.1579 | LR: 9.71e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.3018s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank0 [2026-10-28 16:01:13,234] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:06:10,095] INFO : Step 5800/50000 rank0 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1416 | Classification_binary: 0.2787 | Regression_cos_phi: 0.1719 | Regression_energy: 0.1838 | Regression_eta: 0.1095 | Regression_pt: 0.1898 | Regression_sin_phi: 0.1593 | LR: 9.70e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8803s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank0 [2026-10-28 16:06:10,334] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:11:20,673] INFO : Step 5900/50000 rank0 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1654 | Classification_binary: 0.4105 | Regression_cos_phi: 0.2956 | Regression_energy: 0.1833 | Regression_eta: 0.1617 | Regression_pt: 0.1892 | Regression_sin_phi: 0.2382 | LR: 9.69e-04 | DataLoad Time: 0.0156s | Model Forward Time: 7.9200s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank0 [2026-10-28 16:11:20,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 47.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 94% +4 8 0x7408, 13395 53.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +6 10 0x7408, 62036 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 1% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:16:41,298] INFO : Step 6000/50000 rank0 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1507 | Classification_binary: 0.3601 | Regression_cos_phi: 0.2131 | Regression_energy: 0.1852 | Regression_eta: 0.1679 | Regression_pt: 0.1911 | Regression_sin_phi: 0.2780 | LR: 9.68e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2846s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank0 [2026-10-28 16:16:41,533] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:16:41,645] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 16:16:41,950] INFO : Running validation on rank0 +rank0 [2026-10-28 16:16:41,950] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:16:41,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION | Step=6000/50000 | Train Loss=2.1980 | Valid Loss=2.0482 | Train Loss Unweighted=1.2174 | Valid Loss Unweighted=1.1882 | Stale=0 | ETA=2230.9m +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 16:17:33,111] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari=0.492032 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.397052 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_competing=0.416761 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.555971 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.00630437 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0251658 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:17:33,112] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/seconds_per_rank=22.0492 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 16:17:33,113] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari=0.193959 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding_minus_raw/ari=0.0635188 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 16:17:33,114] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:17:33,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:17:33,377] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:17:33,382] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 16:17:59,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:18:00,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:18:02,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:18:03,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:18:13,310] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:18:13,873] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:18:13,877] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:18:29,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:18:31,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:18:32,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:18:33,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:18:43,384] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 16:18:43,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:18:43,952] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 16:18:43,956] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 16:18:59,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:19:00,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:19:02,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:19:04,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:19:14,125] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 16:19:14,233] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:19:17,448] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits +rank0 [2026-10-28 16:19:18,526] INFO : Plotted number of elements +rank0 [2026-10-28 16:19:20,084] INFO : Plotted jets +rank0 [2026-10-28 16:19:21,632] INFO : Plotted jet ratio +rank0 [2026-10-28 16:19:21,704] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:19:25,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 16:19:26,031] INFO : Plotted number of elements +rank0 [2026-10-28 16:19:27,542] INFO : Plotted jets +rank0 [2026-10-28 16:19:28,683] INFO : Plotted jet ratio +rank0 [2026-10-28 16:19:28,745] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits +rank0 [2026-10-28 16:19:31,254] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits +rank0 [2026-10-28 16:19:32,110] INFO : Plotted number of elements +rank0 [2026-10-28 16:19:33,612] INFO : Plotted jets +rank0 [2026-10-28 16:19:34,808] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:24,397] INFO : Step 6100/50000 rank0 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1129 | Classification_binary: 0.2657 | Regression_cos_phi: 0.1574 | Regression_energy: 0.1668 | Regression_eta: 0.1143 | Regression_pt: 0.1721 | Regression_sin_phi: 0.1509 | LR: 9.67e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.7082s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank0 [2026-10-28 16:24:24,636] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:29:27,573] INFO : Step 6200/50000 rank0 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1273 | Classification_binary: 0.2757 | Regression_cos_phi: 0.1570 | Regression_energy: 0.1618 | Regression_eta: 0.0839 | Regression_pt: 0.1700 | Regression_sin_phi: 0.1558 | LR: 9.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.7733s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank0 [2026-10-28 16:29:27,811] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 61% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 98% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 97% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 11% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 18% +6 10 0x7408, 62036 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 73% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:34:17,998] INFO : Step 6300/50000 rank0 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1361 | Classification_binary: 0.3767 | Regression_cos_phi: 0.1580 | Regression_energy: 0.1562 | Regression_eta: 0.1043 | Regression_pt: 0.1602 | Regression_sin_phi: 0.1773 | LR: 9.65e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.8881s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank0 [2026-10-28 16:34:18,239] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 2% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:39:15,427] INFO : Step 6400/50000 rank0 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1232 | Classification_binary: 0.2710 | Regression_cos_phi: 0.1576 | Regression_energy: 0.1752 | Regression_eta: 0.1107 | Regression_pt: 0.1791 | Regression_sin_phi: 0.1234 | LR: 9.64e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.4818s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank0 [2026-10-28 16:39:15,674] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:43:58,890] INFO : Step 6500/50000 rank0 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1282 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1350 | Regression_energy: 0.1645 | Regression_eta: 0.1175 | Regression_pt: 0.1734 | Regression_sin_phi: 0.1510 | LR: 9.63e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.3296s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank0 [2026-10-28 16:43:59,127] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 3% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:48:53,221] INFO : Step 6600/50000 rank0 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1251 | Classification_binary: 0.3914 | Regression_cos_phi: 0.1779 | Regression_energy: 0.1816 | Regression_eta: 0.1279 | Regression_pt: 0.1899 | Regression_sin_phi: 0.1740 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5435s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank0 [2026-10-28 16:48:53,459] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:53:48,277] INFO : Step 6700/50000 rank0 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2719 | Regression_cos_phi: 0.1077 | Regression_energy: 0.1764 | Regression_eta: 0.1054 | Regression_pt: 0.1829 | Regression_sin_phi: 0.1254 | LR: 9.61e-04 | DataLoad Time: 0.0217s | Model Forward Time: 6.1954s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank0 [2026-10-28 16:53:48,516] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:58:35,132] INFO : Step 6800/50000 rank0 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1086 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1486 | Regression_energy: 0.1604 | Regression_eta: 0.0949 | Regression_pt: 0.1648 | Regression_sin_phi: 0.1450 | LR: 9.60e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.1126s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank0 [2026-10-28 16:58:35,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:03:32,544] INFO : Step 6900/50000 rank0 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3376 | Regression_cos_phi: 0.1672 | Regression_energy: 0.1528 | Regression_eta: 0.1352 | Regression_pt: 0.1602 | Regression_sin_phi: 0.1773 | LR: 9.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.4567s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank0 [2026-10-28 17:03:32,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:08:30,552] INFO : Step 7000/50000 rank0 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1549 | Classification_binary: 0.3478 | Regression_cos_phi: 0.2492 | Regression_energy: 0.1923 | Regression_eta: 0.1580 | Regression_pt: 0.2016 | Regression_sin_phi: 0.1825 | LR: 9.57e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5594s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank0 [2026-10-28 17:08:30,789] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:08:30,911] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 17:08:31,206] INFO : Running validation on rank0 +rank0 [2026-10-28 17:08:31,206] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:08:31,206] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:09:24,246] INFO : VALIDATION | Step=7000/50000 | Train Loss=2.2196 | Valid Loss=1.9521 | Train Loss Unweighted=1.2293 | Valid Loss Unweighted=1.1139 | Stale=0 | ETA=2187.2m +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 17:09:24,247] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari=0.483987 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.415546 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_competing=0.439977 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.541098 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00174029 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0102929 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 17:09:24,248] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/seconds_per_rank=22.1695 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:09:24,249] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari=0.203975 +rank0 [2026-10-28 17:09:24,250] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 17:09:24,250] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding_minus_raw/ari=0.0735351 +rank0 [2026-10-28 17:09:24,250] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 17:09:24,250] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 17:09:24,250] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 17:09:24,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:09:24,508] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 17:09:24,512] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 17:09:52,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 17:09:54,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 17:09:55,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 17:09:57,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 17:10:07,721] INFO : Time taken to make predictions on device 0 is: 0.31 min +rank0 [2026-10-28 17:10:07,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:10:08,262] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 17:10:08,266] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:10:24,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 17:10:25,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 17:10:26,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 17:10:27,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 17:10:37,852] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 17:10:37,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:10:38,806] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 17:10:38,809] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 17:10:54,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 17:10:55,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 17:10:56,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 17:10:57,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 17:11:07,842] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 17:11:07,961] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits +rank0 [2026-10-28 17:11:11,242] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits +rank0 [2026-10-28 17:11:12,345] INFO : Plotted number of elements +rank0 [2026-10-28 17:11:13,888] INFO : Plotted jets +rank0 [2026-10-28 17:11:15,071] INFO : Plotted jet ratio +rank0 [2026-10-28 17:11:15,141] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:11:18,495] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 17:11:19,540] INFO : Plotted number of elements +rank0 [2026-10-28 17:11:21,095] INFO : Plotted jets +rank0 [2026-10-28 17:11:22,303] INFO : Plotted jet ratio +rank0 [2026-10-28 17:11:22,363] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits +rank0 [2026-10-28 17:11:24,824] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits +rank0 [2026-10-28 17:11:25,685] INFO : Plotted number of elements +rank0 [2026-10-28 17:11:27,686] INFO : Plotted jets +rank0 [2026-10-28 17:11:28,883] INFO : Plotted jet ratio +rank0 [2026-10-28 17:16:28,211] INFO : Step 7100/50000 rank0 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1018 | Classification_binary: 0.2665 | Regression_cos_phi: 0.1228 | Regression_energy: 0.1290 | Regression_eta: 0.0975 | Regression_pt: 0.1314 | Regression_sin_phi: 0.1310 | LR: 9.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.7625s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank0 [2026-10-28 17:16:28,447] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:21:34,747] INFO : Step 7200/50000 rank0 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1234 | Classification_binary: 0.3397 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1332 | Regression_eta: 0.0932 | Regression_pt: 0.1388 | Regression_sin_phi: 0.1575 | LR: 9.55e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0145s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank0 [2026-10-28 17:21:35,021] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 47.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:26:35,865] INFO : Step 7300/50000 rank0 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.3365 | Regression_cos_phi: 0.1402 | Regression_energy: 0.1340 | Regression_eta: 0.1077 | Regression_pt: 0.1400 | Regression_sin_phi: 0.1521 | LR: 9.53e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3076s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank0 [2026-10-28 17:26:36,105] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 92% +2 6 0x7408, 15961 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 1% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 98% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:31:35,310] INFO : Step 7400/50000 rank0 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.2736 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1509 | Regression_eta: 0.0978 | Regression_pt: 0.1568 | Regression_sin_phi: 0.1154 | LR: 9.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2197s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank0 [2026-10-28 17:31:35,548] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 36% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 97% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:36:31,623] INFO : Step 7500/50000 rank0 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.2503 | Regression_cos_phi: 0.1423 | Regression_energy: 0.1492 | Regression_eta: 0.0889 | Regression_pt: 0.1564 | Regression_sin_phi: 0.1272 | LR: 9.51e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.3987s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank0 [2026-10-28 17:36:31,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:41:09,769] INFO : Step 7600/50000 rank0 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1291 | Classification_binary: 0.3401 | Regression_cos_phi: 0.1102 | Regression_energy: 0.1649 | Regression_eta: 0.1170 | Regression_pt: 0.1709 | Regression_sin_phi: 0.1725 | LR: 9.50e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4455s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank0 [2026-10-28 17:41:10,009] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:46:04,050] INFO : Step 7700/50000 rank0 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.3307 | Regression_cos_phi: 0.1610 | Regression_energy: 0.1470 | Regression_eta: 0.1151 | Regression_pt: 0.1537 | Regression_sin_phi: 0.1591 | LR: 9.48e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5887s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank0 [2026-10-28 17:46:04,290] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:51:03,143] INFO : Step 7800/50000 rank0 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1043 | Classification_binary: 0.2633 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1427 | Regression_eta: 0.0810 | Regression_pt: 0.1453 | Regression_sin_phi: 0.1452 | LR: 9.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0049s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank0 [2026-10-28 17:51:03,395] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:56:05,954] INFO : Step 7900/50000 rank0 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1416 | Classification_binary: 0.3305 | Regression_cos_phi: 0.1517 | Regression_energy: 0.1615 | Regression_eta: 0.1315 | Regression_pt: 0.1662 | Regression_sin_phi: 0.1394 | LR: 9.46e-04 | DataLoad Time: 0.0205s | Model Forward Time: 7.1407s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank0 [2026-10-28 17:56:06,194] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 53.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:00:55,967] INFO : Step 8000/50000 rank0 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.3369 | Regression_cos_phi: 0.1932 | Regression_energy: 0.1457 | Regression_eta: 0.1303 | Regression_pt: 0.1534 | Regression_sin_phi: 0.1452 | LR: 9.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1570s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank0 [2026-10-28 18:00:56,226] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:00:56,248] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 18:00:56,536] INFO : Running validation on rank0 +rank0 [2026-10-28 18:00:56,536] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:00:56,536] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:01:48,286] INFO : VALIDATION | Step=8000/50000 | Train Loss=1.9491 | Valid Loss=1.8325 | Train Loss Unweighted=1.0841 | Valid Loss Unweighted=1.0273 | Stale=0 | ETA=2144.4m +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 18:01:48,287] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari=0.468758 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari_threshold=0.675 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.442084 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_competing=0.476667 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.577398 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0169691 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0465929 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:01:48,288] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/seconds_per_rank=21.8033 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:01:48,289] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari=0.225394 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding_minus_raw/ari=0.0949541 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 18:01:48,290] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:01:48,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:01:48,919] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:01:48,930] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 18:02:14,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:02:15,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:02:17,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:02:18,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:02:28,325] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:02:28,338] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:02:28,857] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:02:28,860] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:02:44,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:02:45,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:02:47,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:02:48,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:02:58,268] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:02:58,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:02:58,745] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 18:02:58,763] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 18:03:14,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:03:15,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:03:16,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:03:17,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:03:27,789] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 18:03:27,805] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:03:30,971] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:03:32,095] INFO : Plotted number of elements +rank0 [2026-10-28 18:03:33,644] INFO : Plotted jets +rank0 [2026-10-28 18:03:34,759] INFO : Plotted jet ratio +rank0 [2026-10-28 18:03:34,832] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:03:38,045] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:03:39,118] INFO : Plotted number of elements +rank0 [2026-10-28 18:03:40,714] INFO : Plotted jets +rank0 [2026-10-28 18:03:41,866] INFO : Plotted jet ratio +rank0 [2026-10-28 18:03:41,923] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits +rank0 [2026-10-28 18:03:44,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits +rank0 [2026-10-28 18:03:45,298] INFO : Plotted number of elements +rank0 [2026-10-28 18:03:47,274] INFO : Plotted jets +rank0 [2026-10-28 18:03:48,467] INFO : Plotted jet ratio +rank0 [2026-10-28 18:08:31,208] INFO : Step 8100/50000 rank0 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2206 | Regression_cos_phi: 0.0897 | Regression_energy: 0.1399 | Regression_eta: 0.0763 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0979 | LR: 9.43e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.4762s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank0 [2026-10-28 18:08:31,453] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:13:42,556] INFO : Step 8200/50000 rank0 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1219 | Classification_binary: 0.3289 | Regression_cos_phi: 0.1852 | Regression_energy: 0.1310 | Regression_eta: 0.1080 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1686 | LR: 9.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9598s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank0 [2026-10-28 18:13:42,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 47.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:18:40,295] INFO : Step 8300/50000 rank0 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.3179 | Regression_cos_phi: 0.1444 | Regression_energy: 0.1420 | Regression_eta: 0.1334 | Regression_pt: 0.1449 | Regression_sin_phi: 0.1471 | LR: 9.40e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.4012s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank0 [2026-10-28 18:18:40,539] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:23:43,770] INFO : Step 8400/50000 rank0 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2781 | Regression_cos_phi: 0.0968 | Regression_energy: 0.1289 | Regression_eta: 0.0992 | Regression_pt: 0.1325 | Regression_sin_phi: 0.1365 | LR: 9.39e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8634s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank0 [2026-10-28 18:23:44,011] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:28:52,883] INFO : Step 8500/50000 rank0 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2459 | Regression_cos_phi: 0.1492 | Regression_energy: 0.1226 | Regression_eta: 0.0890 | Regression_pt: 0.1287 | Regression_sin_phi: 0.1372 | LR: 9.37e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2548s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank0 [2026-10-28 18:28:53,123] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:34:10,094] INFO : Step 8600/50000 rank0 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1031 | Classification_binary: 0.3050 | Regression_cos_phi: 0.1919 | Regression_energy: 0.1463 | Regression_eta: 0.1310 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1771 | LR: 9.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3953s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank0 [2026-10-28 18:34:10,357] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:39:15,359] INFO : Step 8700/50000 rank0 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1306 | Classification_binary: 0.3350 | Regression_cos_phi: 0.1238 | Regression_energy: 0.1396 | Regression_eta: 0.1002 | Regression_pt: 0.1431 | Regression_sin_phi: 0.1528 | LR: 9.34e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.4986s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank0 [2026-10-28 18:39:15,603] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:44:20,146] INFO : Step 8800/50000 rank0 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2299 | Regression_cos_phi: 0.1268 | Regression_energy: 0.1173 | Regression_eta: 0.0814 | Regression_pt: 0.1208 | Regression_sin_phi: 0.1309 | LR: 9.33e-04 | DataLoad Time: 0.0169s | Model Forward Time: 7.1413s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank0 [2026-10-28 18:44:20,384] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 4% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +2 6 0x7408, 15961 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 86% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 0% +4 8 0x7408, 13395 53.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 2% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +6 10 0x7408, 62036 51.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 4% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:49:29,167] INFO : Step 8900/50000 rank0 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0943 | Classification_binary: 0.2765 | Regression_cos_phi: 0.1608 | Regression_energy: 0.1395 | Regression_eta: 0.1061 | Regression_pt: 0.1452 | Regression_sin_phi: 0.1839 | LR: 9.31e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4368s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank0 [2026-10-28 18:49:29,418] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:54:54,695] INFO : Step 9000/50000 rank0 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.3446 | Regression_cos_phi: 0.1237 | Regression_energy: 0.1363 | Regression_eta: 0.1105 | Regression_pt: 0.1410 | Regression_sin_phi: 0.1681 | LR: 9.30e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.5634s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank0 [2026-10-28 18:54:54,934] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:54:54,955] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 18:54:55,402] INFO : Running validation on rank0 +rank0 [2026-10-28 18:54:55,402] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:54:55,403] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION | Step=9000/50000 | Train Loss=1.8831 | Valid Loss=1.7142 | Train Loss Unweighted=1.0179 | Valid Loss Unweighted=0.9523 | Stale=0 | ETA=2107.1m +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 18:55:53,216] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari=0.455136 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.428677 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_competing=0.457046 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.594795 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0305915 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0639904 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 18:55:53,217] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/seconds_per_rank=22.3238 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:55:53,218] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari=0.237184 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding_minus_raw/ari=0.106744 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 18:55:53,219] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:55:53,224] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:55:53,479] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:55:53,490] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 18:56:20,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:56:21,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:56:23,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:56:24,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:56:34,820] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-28 18:56:34,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:56:35,475] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:56:35,479] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:56:51,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:56:53,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:56:54,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:56:55,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:57:05,402] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:57:06,114] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 18:57:06,118] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 18:57:21,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:57:22,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:57:23,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:57:24,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:57:34,256] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 18:57:34,484] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:57:37,778] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits +rank0 [2026-10-28 18:57:38,960] INFO : Plotted number of elements +rank0 [2026-10-28 18:57:40,612] INFO : Plotted jets +rank0 [2026-10-28 18:57:41,869] INFO : Plotted jet ratio +rank0 [2026-10-28 18:57:41,920] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:57:45,223] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 18:57:46,278] INFO : Plotted number of elements +rank0 [2026-10-28 18:57:47,857] INFO : Plotted jets +rank0 [2026-10-28 18:57:49,052] INFO : Plotted jet ratio +rank0 [2026-10-28 18:57:49,109] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits +rank0 [2026-10-28 18:57:51,587] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits +rank0 [2026-10-28 18:57:52,441] INFO : Plotted number of elements +rank0 [2026-10-28 18:57:54,427] INFO : Plotted jets +rank0 [2026-10-28 18:57:55,632] INFO : Plotted jet ratio +rank0 [2026-10-28 19:03:09,508] INFO : Step 9100/50000 rank0 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2196 | Regression_cos_phi: 0.1057 | Regression_energy: 0.1100 | Regression_eta: 0.0756 | Regression_pt: 0.1105 | Regression_sin_phi: 0.1108 | LR: 9.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.0790s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank0 [2026-10-28 19:03:09,751] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 50.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:08:19,103] INFO : Step 9200/50000 rank0 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2541 | Regression_cos_phi: 0.1131 | Regression_energy: 0.1521 | Regression_eta: 0.1087 | Regression_pt: 0.1572 | Regression_sin_phi: 0.1302 | LR: 9.27e-04 | DataLoad Time: 0.0249s | Model Forward Time: 7.3458s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank0 [2026-10-28 19:08:19,354] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:13:31,112] INFO : Step 9300/50000 rank0 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0954 | Classification_binary: 0.3011 | Regression_cos_phi: 0.1227 | Regression_energy: 0.1357 | Regression_eta: 0.0737 | Regression_pt: 0.1421 | Regression_sin_phi: 0.1271 | LR: 9.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1847s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank0 [2026-10-28 19:13:31,362] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:18:56,381] INFO : Step 9400/50000 rank0 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2628 | Regression_cos_phi: 0.1886 | Regression_energy: 0.1470 | Regression_eta: 0.1126 | Regression_pt: 0.1495 | Regression_sin_phi: 0.1420 | LR: 9.24e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9689s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank0 [2026-10-28 19:18:56,625] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 9% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 2% +2 6 0x7408, 15961 47.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 38% +4 8 0x7408, 13395 54.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 2% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 19% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:24:21,590] INFO : Step 9500/50000 rank0 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0983 | Classification_binary: 0.2046 | Regression_cos_phi: 0.1222 | Regression_energy: 0.1287 | Regression_eta: 0.1034 | Regression_pt: 0.1319 | Regression_sin_phi: 0.1224 | LR: 9.22e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.0823s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank0 [2026-10-28 19:24:21,827] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:29:35,074] INFO : Step 9600/50000 rank0 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1030 | Classification_binary: 0.3122 | Regression_cos_phi: 0.1593 | Regression_energy: 0.1286 | Regression_eta: 0.0888 | Regression_pt: 0.1343 | Regression_sin_phi: 0.1620 | LR: 9.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.2386s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank0 [2026-10-28 19:29:35,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 56.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:34:54,535] INFO : Step 9700/50000 rank0 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0928 | Classification_binary: 0.2885 | Regression_cos_phi: 0.1383 | Regression_energy: 0.1299 | Regression_eta: 0.1033 | Regression_pt: 0.1350 | Regression_sin_phi: 0.1437 | LR: 9.19e-04 | DataLoad Time: 0.0097s | Model Forward Time: 5.7538s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank0 [2026-10-28 19:34:54,777] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:39:54,654] INFO : Step 9800/50000 rank0 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.2004 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0965 | Regression_eta: 0.0684 | Regression_pt: 0.1020 | Regression_sin_phi: 0.1121 | LR: 9.17e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.2783s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank0 [2026-10-28 19:39:54,899] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:44:47,303] INFO : Step 9900/50000 rank0 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2872 | Regression_cos_phi: 0.1494 | Regression_energy: 0.1023 | Regression_eta: 0.1001 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1529 | LR: 9.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6859s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank0 [2026-10-28 19:44:47,545] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:49:38,445] INFO : Step 10000/50000 rank0 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.3280 | Regression_cos_phi: 0.1639 | Regression_energy: 0.1210 | Regression_eta: 0.1165 | Regression_pt: 0.1259 | Regression_sin_phi: 0.1341 | LR: 9.14e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.4349s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank0 [2026-10-28 19:49:38,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 3% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 47.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 54.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 46% 0% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 94% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:49:38,711] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 19:49:39,086] INFO : Running validation on rank0 +rank0 [2026-10-28 19:49:39,086] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 19:49:39,087] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION | Step=10000/50000 | Train Loss=1.7651 | Valid Loss=1.6149 | Train Loss Unweighted=0.9525 | Valid Loss Unweighted=0.8842 | Stale=0 | ETA=2069.0m +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 19:50:35,646] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari=0.490609 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.474488 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_competing=0.514199 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.594484 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.00488208 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0636793 +rank0 [2026-10-28 19:50:35,647] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/seconds_per_rank=22.4282 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:50:35,648] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari=0.24179 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding_minus_raw/ari=0.111351 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 19:50:35,649] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 19:50:35,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:50:35,906] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 19:50:35,916] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 19:51:01,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 19:51:03,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 19:51:04,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 19:51:05,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 19:51:15,835] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 19:51:15,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:51:16,369] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 19:51:16,372] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:51:32,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 19:51:33,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 19:51:35,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 19:51:36,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 19:51:46,129] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 19:51:46,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:51:46,686] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 19:51:46,690] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 19:52:02,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 19:52:03,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 19:52:04,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 19:52:05,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 19:52:15,797] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 19:52:15,814] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits +rank0 [2026-10-28 19:52:19,011] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits +rank0 [2026-10-28 19:52:20,138] INFO : Plotted number of elements +rank0 [2026-10-28 19:52:21,618] INFO : Plotted jets +rank0 [2026-10-28 19:52:22,799] INFO : Plotted jet ratio +rank0 [2026-10-28 19:52:22,849] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:52:26,067] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 19:52:27,098] INFO : Plotted number of elements +rank0 [2026-10-28 19:52:28,598] INFO : Plotted jets +rank0 [2026-10-28 19:52:29,745] INFO : Plotted jet ratio +rank0 [2026-10-28 19:52:29,797] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits +rank0 [2026-10-28 19:52:32,658] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits +rank0 [2026-10-28 19:52:33,453] INFO : Plotted number of elements +rank0 [2026-10-28 19:52:35,044] INFO : Plotted jets +rank0 [2026-10-28 19:52:36,365] INFO : Plotted jet ratio +rank0 [2026-10-28 19:57:28,872] INFO : Step 10100/50000 rank0 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.2757 | Regression_cos_phi: 0.1073 | Regression_energy: 0.1050 | Regression_eta: 0.0823 | Regression_pt: 0.1110 | Regression_sin_phi: 0.1181 | LR: 9.12e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6491s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank0 [2026-10-28 19:57:29,113] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:02:16,168] INFO : Step 10200/50000 rank0 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.2820 | Regression_cos_phi: 0.1763 | Regression_energy: 0.1250 | Regression_eta: 0.0936 | Regression_pt: 0.1295 | Regression_sin_phi: 0.1123 | LR: 9.11e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.3060s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank0 [2026-10-28 20:02:16,410] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:07:32,524] INFO : Step 10300/50000 rank0 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.3082 | Regression_cos_phi: 0.1706 | Regression_energy: 0.1213 | Regression_eta: 0.1260 | Regression_pt: 0.1227 | Regression_sin_phi: 0.1426 | LR: 9.09e-04 | DataLoad Time: 0.0231s | Model Forward Time: 5.9698s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank0 [2026-10-28 20:07:32,764] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:12:49,842] INFO : Step 10400/50000 rank0 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2376 | Regression_cos_phi: 0.0943 | Regression_energy: 0.1029 | Regression_eta: 0.0609 | Regression_pt: 0.1083 | Regression_sin_phi: 0.1099 | LR: 9.07e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.9137s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank0 [2026-10-28 20:12:50,081] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +2 6 0x7408, 15961 47.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 80% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +4 8 0x7408, 13395 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 26% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:18:09,998] INFO : Step 10500/50000 rank0 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0958 | Regression_energy: 0.1345 | Regression_eta: 0.0689 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0875 | LR: 9.06e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.1673s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank0 [2026-10-28 20:18:10,240] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 95% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 55% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 5% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 89% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 13% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 41% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:23:17,778] INFO : Step 10600/50000 rank0 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0943 | Classification_binary: 0.2785 | Regression_cos_phi: 0.1408 | Regression_energy: 0.1046 | Regression_eta: 0.1067 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1470 | LR: 9.04e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.8112s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank0 [2026-10-28 20:23:18,035] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:28:23,401] INFO : Step 10700/50000 rank0 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1242 | Classification_binary: 0.3130 | Regression_cos_phi: 0.1567 | Regression_energy: 0.1083 | Regression_eta: 0.0898 | Regression_pt: 0.1112 | Regression_sin_phi: 0.1157 | LR: 9.02e-04 | DataLoad Time: 0.0226s | Model Forward Time: 5.5601s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank0 [2026-10-28 20:28:23,639] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:33:29,622] INFO : Step 10800/50000 rank0 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0968 | Regression_eta: 0.0656 | Regression_pt: 0.0977 | Regression_sin_phi: 0.1074 | LR: 9.00e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.3087s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank0 [2026-10-28 20:33:29,861] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:38:44,763] INFO : Step 10900/50000 rank0 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1108 | Regression_eta: 0.0693 | Regression_pt: 0.1158 | Regression_sin_phi: 0.1345 | LR: 8.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4069s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank0 [2026-10-28 20:38:45,003] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 56.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:43:47,176] INFO : Step 11000/50000 rank0 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1063 | Classification_binary: 0.2957 | Regression_cos_phi: 0.1138 | Regression_energy: 0.0981 | Regression_eta: 0.0894 | Regression_pt: 0.1035 | Regression_sin_phi: 0.1175 | LR: 8.97e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0912s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank0 [2026-10-28 20:43:47,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:43:47,437] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 20:43:47,836] INFO : Running validation on rank0 +rank0 [2026-10-28 20:43:47,836] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 20:43:47,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION | Step=11000/50000 | Train Loss=1.6462 | Valid Loss=1.4640 | Train Loss Unweighted=0.8747 | Valid Loss Unweighted=0.7848 | Stale=0 | ETA=2025.7m +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 20:44:41,886] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari=0.514742 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.506013 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_competing=0.553257 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.587739 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.029015 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0569347 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 20:44:41,887] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/seconds_per_rank=21.959 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari=0.251297 +rank0 [2026-10-28 20:44:41,888] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 20:44:41,889] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding_minus_raw/ari=0.120858 +rank0 [2026-10-28 20:44:41,889] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 20:44:41,889] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 20:44:41,889] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 20:44:41,889] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:44:42,142] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 20:44:42,148] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 20:45:10,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 20:45:11,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 20:45:12,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 20:45:13,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 20:45:23,522] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 20:45:23,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:45:24,126] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 20:45:24,130] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:45:40,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 20:45:41,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 20:45:43,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 20:45:44,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 20:45:54,432] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:45:55,828] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 20:45:55,839] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 20:46:11,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 20:46:12,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 20:46:14,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 20:46:15,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 20:46:25,941] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 20:46:25,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits +rank0 [2026-10-28 20:46:29,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits +rank0 [2026-10-28 20:46:30,532] INFO : Plotted number of elements +rank0 [2026-10-28 20:46:32,168] INFO : Plotted jets +rank0 [2026-10-28 20:46:33,873] INFO : Plotted jet ratio +rank0 [2026-10-28 20:46:33,925] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:46:37,369] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 20:46:38,430] INFO : Plotted number of elements +rank0 [2026-10-28 20:46:39,941] INFO : Plotted jets +rank0 [2026-10-28 20:46:41,107] INFO : Plotted jet ratio +rank0 [2026-10-28 20:46:41,158] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits +rank0 [2026-10-28 20:46:43,657] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits +rank0 [2026-10-28 20:46:44,517] INFO : Plotted number of elements +rank0 [2026-10-28 20:46:46,024] INFO : Plotted jets +rank0 [2026-10-28 20:46:47,214] INFO : Plotted jet ratio +rank0 [2026-10-28 20:51:57,054] INFO : Step 11100/50000 rank0 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1873 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1158 | Regression_eta: 0.0709 | Regression_pt: 0.1182 | Regression_sin_phi: 0.1447 | LR: 8.95e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.9174s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank0 [2026-10-28 20:51:57,295] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 84% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:57:10,700] INFO : Step 11200/50000 rank0 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2077 | Regression_cos_phi: 0.1077 | Regression_energy: 0.0900 | Regression_eta: 0.0732 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0886 | LR: 8.93e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.4611s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank0 [2026-10-28 20:57:10,938] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:02:19,237] INFO : Step 11300/50000 rank0 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.3152 | Regression_cos_phi: 0.1601 | Regression_energy: 0.1166 | Regression_eta: 0.0951 | Regression_pt: 0.1207 | Regression_sin_phi: 0.1610 | LR: 8.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.0674s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank0 [2026-10-28 21:02:19,476] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:07:20,905] INFO : Step 11400/50000 rank0 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0960 | Classification_binary: 0.3033 | Regression_cos_phi: 0.1315 | Regression_energy: 0.0933 | Regression_eta: 0.0880 | Regression_pt: 0.0979 | Regression_sin_phi: 0.1332 | LR: 8.89e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.6439s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank0 [2026-10-28 21:07:21,142] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:12:16,707] INFO : Step 11500/50000 rank0 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0993 | Regression_energy: 0.1323 | Regression_eta: 0.0804 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1230 | LR: 8.88e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1063s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank0 [2026-10-28 21:12:16,948] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:17:17,953] INFO : Step 11600/50000 rank0 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2376 | Regression_cos_phi: 0.0926 | Regression_energy: 0.1113 | Regression_eta: 0.0542 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0817 | LR: 8.86e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0153s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank0 [2026-10-28 21:17:18,196] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 54.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 98% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:22:14,071] INFO : Step 11700/50000 rank0 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2947 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1186 | Regression_eta: 0.0843 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1429 | LR: 8.84e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2019s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank0 [2026-10-28 21:22:14,308] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:27:08,479] INFO : Step 11800/50000 rank0 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2264 | Regression_cos_phi: 0.1220 | Regression_energy: 0.0962 | Regression_eta: 0.0674 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0763 | LR: 8.82e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1874s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank0 [2026-10-28 21:27:08,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:31:58,610] INFO : Step 11900/50000 rank0 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1062 | Regression_energy: 0.1079 | Regression_eta: 0.0941 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0893 | LR: 8.80e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.6542s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank0 [2026-10-28 21:31:58,893] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:36:38,101] INFO : Step 12000/50000 rank0 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2770 | Regression_cos_phi: 0.1380 | Regression_energy: 0.0902 | Regression_eta: 0.0918 | Regression_pt: 0.0935 | Regression_sin_phi: 0.1504 | LR: 8.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.8623s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank0 [2026-10-28 21:36:38,360] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:36:38,391] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 21:36:39,121] INFO : Running validation on rank0 +rank0 [2026-10-28 21:36:39,122] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:36:39,122] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:37:31,949] INFO : VALIDATION | Step=12000/50000 | Train Loss=1.5318 | Valid Loss=1.3825 | Train Loss Unweighted=0.8269 | Valid Loss Unweighted=0.7428 | Stale=0 | ETA=1976.6m +rank0 [2026-10-28 21:37:31,949] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari=0.496706 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 21:37:31,950] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.51899 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_competing=0.569314 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.57643 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0109787 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0456251 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 21:37:31,951] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/seconds_per_rank=21.8605 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari=0.26181 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding_minus_raw/ari=0.13137 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 21:37:31,952] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:37:31,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:37:32,215] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:37:32,222] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 21:38:00,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:38:01,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:38:02,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:38:04,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:38:14,062] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:38:14,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:38:14,611] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:38:14,614] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:38:30,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:38:33,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:38:34,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:38:35,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:38:45,645] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 21:38:45,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:38:46,116] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 21:38:46,120] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 21:39:02,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:39:03,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:39:04,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:39:05,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:39:15,518] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 21:39:15,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:39:18,685] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits +rank0 [2026-10-28 21:39:19,763] INFO : Plotted number of elements +rank0 [2026-10-28 21:39:21,320] INFO : Plotted jets +rank0 [2026-10-28 21:39:22,510] INFO : Plotted jet ratio +rank0 [2026-10-28 21:39:22,566] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:39:25,781] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 21:39:26,761] INFO : Plotted number of elements +rank0 [2026-10-28 21:39:28,270] INFO : Plotted jets +rank0 [2026-10-28 21:39:29,433] INFO : Plotted jet ratio +rank0 [2026-10-28 21:39:29,487] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits +rank0 [2026-10-28 21:39:31,936] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits +rank0 [2026-10-28 21:39:32,818] INFO : Plotted number of elements +rank0 [2026-10-28 21:39:34,801] INFO : Plotted jets +rank0 [2026-10-28 21:39:35,987] INFO : Plotted jet ratio +rank0 [2026-10-28 21:44:18,644] INFO : Step 12100/50000 rank0 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2368 | Regression_cos_phi: 0.1267 | Regression_energy: 0.0959 | Regression_eta: 0.0697 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0900 | LR: 8.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.0772s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank0 [2026-10-28 21:44:18,882] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:49:01,640] INFO : Step 12200/50000 rank0 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0999 | Regression_energy: 0.1002 | Regression_eta: 0.0773 | Regression_pt: 0.1043 | Regression_sin_phi: 0.1314 | LR: 8.74e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.1517s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank0 [2026-10-28 21:49:01,897] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:53:56,652] INFO : Step 12300/50000 rank0 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2969 | Regression_cos_phi: 0.1310 | Regression_energy: 0.0971 | Regression_eta: 0.0733 | Regression_pt: 0.1008 | Regression_sin_phi: 0.1383 | LR: 8.72e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4583s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank0 [2026-10-28 21:53:56,893] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:58:41,799] INFO : Step 12400/50000 rank0 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2604 | Regression_cos_phi: 0.0975 | Regression_energy: 0.1040 | Regression_eta: 0.0596 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0754 | LR: 8.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.0562s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank0 [2026-10-28 21:58:42,056] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:03:24,065] INFO : Step 12500/50000 rank0 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0886 | Regression_eta: 0.0829 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0824 | LR: 8.68e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3498s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank0 [2026-10-28 22:03:24,302] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:08:05,641] INFO : Step 12600/50000 rank0 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2428 | Regression_cos_phi: 0.1305 | Regression_energy: 0.1036 | Regression_eta: 0.0689 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0892 | LR: 8.66e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.9337s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank0 [2026-10-28 22:08:05,884] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 80% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:12:51,910] INFO : Step 12700/50000 rank0 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2858 | Regression_cos_phi: 0.1257 | Regression_energy: 0.0946 | Regression_eta: 0.0870 | Regression_pt: 0.0999 | Regression_sin_phi: 0.1064 | LR: 8.64e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0849s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank0 [2026-10-28 22:12:52,148] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:17:39,199] INFO : Step 12800/50000 rank0 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2300 | Regression_cos_phi: 0.1483 | Regression_energy: 0.1174 | Regression_eta: 0.0891 | Regression_pt: 0.1198 | Regression_sin_phi: 0.1165 | LR: 8.62e-04 | DataLoad Time: 0.0241s | Model Forward Time: 6.7561s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank0 [2026-10-28 22:17:39,451] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:22:33,383] INFO : Step 12900/50000 rank0 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2020 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0909 | Regression_eta: 0.0456 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0890 | LR: 8.60e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3045s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank0 [2026-10-28 22:22:33,636] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:27:22,692] INFO : Step 13000/50000 rank0 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0940 | Classification_binary: 0.2771 | Regression_cos_phi: 0.1330 | Regression_energy: 0.1095 | Regression_eta: 0.1059 | Regression_pt: 0.1118 | Regression_sin_phi: 0.1315 | LR: 8.58e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.7216s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank0 [2026-10-28 22:27:22,942] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:27:24,030] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 22:27:25,283] INFO : Running validation on rank0 +rank0 [2026-10-28 22:27:25,283] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 22:27:25,283] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION | Step=13000/50000 | Train Loss=1.4749 | Valid Loss=1.3046 | Train Loss Unweighted=0.7377 | Valid Loss Unweighted=0.6863 | Stale=0 | ETA=1921.5m +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:28:28,146] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari=0.505492 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.567512 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_competing=0.629365 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.605089 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0197645 +rank0 [2026-10-28 22:28:28,147] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.074284 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/seconds_per_rank=22.1495 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 22:28:28,148] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari=0.297108 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding_minus_raw/ari=0.166668 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 22:28:28,149] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 22:28:28,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:28:28,427] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 22:28:28,437] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 22:28:54,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 22:28:55,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 22:28:56,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 22:28:58,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 22:29:08,044] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 22:29:08,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:29:08,605] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 22:29:08,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:29:24,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 22:29:25,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 22:29:26,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 22:29:27,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 22:29:37,845] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:29:38,497] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 22:29:38,500] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 22:29:54,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 22:29:55,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 22:29:56,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 22:29:57,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 22:30:07,492] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 22:30:07,510] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits +rank0 [2026-10-28 22:30:10,753] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits +rank0 [2026-10-28 22:30:11,856] INFO : Plotted number of elements +rank0 [2026-10-28 22:30:13,413] INFO : Plotted jets +rank0 [2026-10-28 22:30:14,541] INFO : Plotted jet ratio +rank0 [2026-10-28 22:30:14,642] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:30:17,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 22:30:18,961] INFO : Plotted number of elements +rank0 [2026-10-28 22:30:20,548] INFO : Plotted jets +rank0 [2026-10-28 22:30:22,205] INFO : Plotted jet ratio +rank0 [2026-10-28 22:30:22,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits +rank0 [2026-10-28 22:30:24,638] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits +rank0 [2026-10-28 22:30:25,490] INFO : Plotted number of elements +rank0 [2026-10-28 22:30:27,007] INFO : Plotted jets +rank0 [2026-10-28 22:30:28,209] INFO : Plotted jet ratio +rank0 [2026-10-28 22:35:16,631] INFO : Step 13100/50000 rank0 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0603 | Regression_energy: 0.1010 | Regression_eta: 0.0489 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0668 | LR: 8.56e-04 | DataLoad Time: 0.0222s | Model Forward Time: 7.3200s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank0 [2026-10-28 22:35:16,891] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +4 8 0x7408, 13395 53.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:40:04,770] INFO : Step 13200/50000 rank0 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1847 | Regression_cos_phi: 0.1137 | Regression_energy: 0.0988 | Regression_eta: 0.0484 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0694 | LR: 8.54e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3834s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank0 [2026-10-28 22:40:05,019] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:44:59,344] INFO : Step 13300/50000 rank0 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2469 | Regression_cos_phi: 0.0958 | Regression_energy: 0.1040 | Regression_eta: 0.0693 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0951 | LR: 8.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.4797s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank0 [2026-10-28 22:44:59,582] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:49:50,844] INFO : Step 13400/50000 rank0 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.3215 | Regression_cos_phi: 0.1303 | Regression_energy: 0.1097 | Regression_eta: 0.0912 | Regression_pt: 0.1123 | Regression_sin_phi: 0.1144 | LR: 8.50e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.2541s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank0 [2026-10-28 22:49:51,084] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:54:40,217] INFO : Step 13500/50000 rank0 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0975 | Regression_eta: 0.0496 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0984 | LR: 8.48e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1681s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank0 [2026-10-28 22:54:40,459] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:59:25,845] INFO : Step 13600/50000 rank0 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0835 | Regression_eta: 0.0513 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0885 | LR: 8.45e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.5623s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank0 [2026-10-28 22:59:26,102] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:04:06,676] INFO : Step 13700/50000 rank0 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2414 | Regression_cos_phi: 0.0790 | Regression_energy: 0.1058 | Regression_eta: 0.0491 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0845 | LR: 8.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.4186s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank0 [2026-10-28 23:04:06,916] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 6% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 6% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:09:17,073] INFO : Step 13800/50000 rank0 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0957 | Regression_energy: 0.1076 | Regression_eta: 0.0592 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1101 | LR: 8.41e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.2573s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank0 [2026-10-28 23:09:17,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:14:10,790] INFO : Step 13900/50000 rank0 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0871 | Regression_energy: 0.1047 | Regression_eta: 0.0523 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0674 | LR: 8.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9256s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank0 [2026-10-28 23:14:11,029] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:19:00,588] INFO : Step 14000/50000 rank0 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0935 | Classification_binary: 0.2860 | Regression_cos_phi: 0.1339 | Regression_energy: 0.1018 | Regression_eta: 0.0827 | Regression_pt: 0.1054 | Regression_sin_phi: 0.1075 | LR: 8.37e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.0412s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank0 [2026-10-28 23:19:00,827] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 47.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:19:00,883] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 23:19:01,170] INFO : Running validation on rank0 +rank0 [2026-10-28 23:19:01,170] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:19:01,170] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION | Step=14000/50000 | Train Loss=1.5016 | Valid Loss=1.2856 | Train Loss Unweighted=0.7565 | Valid Loss Unweighted=0.6644 | Stale=0 | ETA=1868.3m +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:19:53,510] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari=0.524605 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.574545 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_competing=0.638299 +rank0 [2026-10-28 23:19:53,511] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.613956 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0388779 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0831513 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-28 23:19:53,512] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/seconds_per_rank=21.8142 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari=0.287675 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding_minus_raw/ari=0.157235 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-28 23:19:53,513] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 23:19:53,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:19:53,782] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-28 23:19:53,788] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-28 23:20:19,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 23:20:20,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 23:20:21,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 23:20:23,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 23:20:33,189] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 23:20:33,202] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:20:33,744] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 23:20:33,748] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:20:50,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 23:20:51,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 23:20:52,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 23:20:53,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 23:21:03,441] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 23:21:03,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:21:04,037] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-28 23:21:04,041] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-28 23:21:19,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 23:21:20,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 23:21:21,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 23:21:23,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 23:21:33,089] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 23:21:33,105] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits +rank0 [2026-10-28 23:21:36,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits +rank0 [2026-10-28 23:21:37,356] INFO : Plotted number of elements +rank0 [2026-10-28 23:21:38,869] INFO : Plotted jets +rank0 [2026-10-28 23:21:39,993] INFO : Plotted jet ratio +rank0 [2026-10-28 23:21:40,067] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:21:43,357] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits +rank0 [2026-10-28 23:21:44,367] INFO : Plotted number of elements +rank0 [2026-10-28 23:21:45,878] INFO : Plotted jets +rank0 [2026-10-28 23:21:47,036] INFO : Plotted jet ratio +rank0 [2026-10-28 23:21:47,087] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits +rank0 [2026-10-28 23:21:49,582] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits +rank0 [2026-10-28 23:21:50,449] INFO : Plotted number of elements +rank0 [2026-10-28 23:21:52,450] INFO : Plotted jets +rank0 [2026-10-28 23:21:53,641] INFO : Plotted jet ratio +rank0 [2026-10-28 23:26:44,410] INFO : Step 14100/50000 rank0 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2403 | Regression_cos_phi: 0.1001 | Regression_energy: 0.0733 | Regression_eta: 0.0769 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0926 | LR: 8.35e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.4945s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank0 [2026-10-28 23:26:44,652] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:31:32,406] INFO : Step 14200/50000 rank0 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0747 | Regression_eta: 0.0359 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0641 | LR: 8.32e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3016s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank0 [2026-10-28 23:31:32,648] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:36:17,348] INFO : Step 14300/50000 rank0 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0711 | Regression_eta: 0.0409 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0770 | LR: 8.30e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3815s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank0 [2026-10-28 23:36:17,612] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:40:59,070] INFO : Step 14400/50000 rank0 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2646 | Regression_cos_phi: 0.1086 | Regression_energy: 0.0898 | Regression_eta: 0.0789 | Regression_pt: 0.0926 | Regression_sin_phi: 0.1173 | LR: 8.28e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.5072s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank0 [2026-10-28 23:40:59,312] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 5% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 67% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:45:43,570] INFO : Step 14500/50000 rank0 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0785 | Regression_eta: 0.0618 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0593 | LR: 8.26e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.2637s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank0 [2026-10-28 23:45:43,807] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:50:28,157] INFO : Step 14600/50000 rank0 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1884 | Regression_cos_phi: 0.1013 | Regression_energy: 0.0766 | Regression_eta: 0.0480 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0849 | LR: 8.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1101s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank0 [2026-10-28 23:50:28,399] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 47.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:55:13,499] INFO : Step 14700/50000 rank0 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.2818 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0756 | Regression_eta: 0.0571 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0891 | LR: 8.21e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.3020s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank0 [2026-10-28 23:55:13,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 47.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:59:57,223] INFO : Step 14800/50000 rank0 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0880 | Regression_eta: 0.0690 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0947 | LR: 8.19e-04 | DataLoad Time: 0.0208s | Model Forward Time: 6.4981s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank0 [2026-10-28 23:59:57,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 47.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:04:47,049] INFO : Step 14900/50000 rank0 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0961 | Regression_eta: 0.0492 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0605 | LR: 8.17e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.8053s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank0 [2026-10-29 00:04:47,288] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 46% 0% +6 10 0x7408, 62036 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:09:43,906] INFO : Step 15000/50000 rank0 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2171 | Regression_cos_phi: 0.1169 | Regression_energy: 0.0843 | Regression_eta: 0.0657 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0971 | LR: 8.15e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.5310s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank0 [2026-10-29 00:09:44,169] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:09:44,192] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-15000.pth +rank0 [2026-10-29 00:09:44,485] INFO : Running validation on rank0 +rank0 [2026-10-29 00:09:44,486] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 00:09:44,486] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION | Step=15000/50000 | Train Loss=1.3450 | Valid Loss=1.2879 | Train Loss Unweighted=0.6824 | Valid Loss Unweighted=0.6541 | Stale=1 | ETA=1813.7m +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:10:39,271] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari=0.516172 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 00:10:39,272] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.576408 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_competing=0.639888 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.621319 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0304447 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0905143 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 00:10:39,273] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/seconds_per_rank=22.0558 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 00:10:39,274] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari=0.274348 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding_minus_raw/ari=0.143909 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 00:10:39,275] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:10:39,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:10:39,538] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:10:39,541] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 00:11:05,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:11:07,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:11:08,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:11:09,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:11:19,648] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 00:11:20,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:11:20,775] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:11:20,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:11:36,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:11:38,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:11:39,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:11:40,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:11:50,433] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 00:11:50,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:11:50,984] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 00:11:50,988] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 00:12:06,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:12:07,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:12:08,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:12:09,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:12:19,638] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 00:12:19,659] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:12:22,973] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits +rank0 [2026-10-29 00:12:24,102] INFO : Plotted number of elements +rank0 [2026-10-29 00:12:25,659] INFO : Plotted jets +rank0 [2026-10-29 00:12:26,801] INFO : Plotted jet ratio +rank0 [2026-10-29 00:12:26,869] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:12:30,161] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 00:12:31,166] INFO : Plotted number of elements +rank0 [2026-10-29 00:12:33,124] INFO : Plotted jets +rank0 [2026-10-29 00:12:34,285] INFO : Plotted jet ratio +rank0 [2026-10-29 00:12:34,336] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits +rank0 [2026-10-29 00:12:36,688] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits +rank0 [2026-10-29 00:12:37,540] INFO : Plotted number of elements +rank0 [2026-10-29 00:12:39,083] INFO : Plotted jets +rank0 [2026-10-29 00:12:40,314] INFO : Plotted jet ratio +rank0 [2026-10-29 00:17:31,172] INFO : Step 15100/50000 rank0 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0985 | Regression_energy: 0.0714 | Regression_eta: 0.0504 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0922 | LR: 8.12e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.3172s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank0 [2026-10-29 00:17:31,416] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:22:17,702] INFO : Step 15200/50000 rank0 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.1952 | Regression_cos_phi: 0.1053 | Regression_energy: 0.0963 | Regression_eta: 0.0521 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0697 | LR: 8.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.0060s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank0 [2026-10-29 00:22:17,943] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 1% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:27:06,469] INFO : Step 15300/50000 rank0 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2244 | Regression_cos_phi: 0.1346 | Regression_energy: 0.1012 | Regression_eta: 0.0695 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0971 | LR: 8.08e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0653s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank0 [2026-10-29 00:27:06,713] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:31:58,386] INFO : Step 15400/50000 rank0 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.2455 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0895 | Regression_eta: 0.0594 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0943 | LR: 8.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3253s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank0 [2026-10-29 00:31:58,625] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:36:53,412] INFO : Step 15500/50000 rank0 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0667 | Regression_eta: 0.0707 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0898 | LR: 8.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0432s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank0 [2026-10-29 00:36:53,653] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:41:49,906] INFO : Step 15600/50000 rank0 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0827 | Regression_eta: 0.0436 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0788 | LR: 8.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5672s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank0 [2026-10-29 00:41:50,170] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:46:44,518] INFO : Step 15700/50000 rank0 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2184 | Regression_cos_phi: 0.1474 | Regression_energy: 0.0831 | Regression_eta: 0.0506 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0844 | LR: 7.98e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.3940s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank0 [2026-10-29 00:46:44,783] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:51:30,481] INFO : Step 15800/50000 rank0 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2470 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0812 | Regression_eta: 0.0742 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0899 | LR: 7.96e-04 | DataLoad Time: 0.0245s | Model Forward Time: 5.8083s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank0 [2026-10-29 00:51:30,723] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:56:25,626] INFO : Step 15900/50000 rank0 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0864 | Regression_eta: 0.0433 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0799 | LR: 7.94e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7998s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank0 [2026-10-29 00:56:25,885] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:01:17,638] INFO : Step 16000/50000 rank0 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0727 | Regression_eta: 0.0548 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0611 | LR: 7.91e-04 | DataLoad Time: 0.0213s | Model Forward Time: 6.1160s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank0 [2026-10-29 01:01:17,881] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:01:17,904] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-16000.pth +rank0 [2026-10-29 01:01:18,311] INFO : Running validation on rank0 +rank0 [2026-10-29 01:01:18,311] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 01:01:18,311] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION | Step=16000/50000 | Train Loss=1.2559 | Valid Loss=1.2106 | Train Loss Unweighted=0.6307 | Valid Loss Unweighted=0.6164 | Stale=0 | ETA=1761.3m +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 01:02:11,365] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari=0.542698 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.62209 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_competing=0.696821 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.619106 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0569704 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0883013 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 01:02:11,366] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/seconds_per_rank=22.1983 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 01:02:11,367] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari=0.301926 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding_minus_raw/ari=0.171486 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 01:02:11,368] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 01:02:11,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:02:11,629] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 01:02:11,639] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 01:02:39,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 01:02:40,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 01:02:43,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 01:02:44,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 01:02:55,008] INFO : Time taken to make predictions on device 0 is: 0.30 min +rank0 [2026-10-29 01:02:55,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:02:55,636] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 01:02:55,640] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:03:11,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 01:03:12,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 01:03:14,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 01:03:15,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 01:03:25,523] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 01:03:25,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:03:26,084] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 01:03:26,087] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 01:03:41,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 01:03:42,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 01:03:43,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 01:03:44,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 01:03:54,569] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 01:03:54,636] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:03:57,957] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:03:59,104] INFO : Plotted number of elements +rank0 [2026-10-29 01:04:00,700] INFO : Plotted jets +rank0 [2026-10-29 01:04:01,886] INFO : Plotted jet ratio +rank0 [2026-10-29 01:04:01,965] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:04:05,261] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:04:06,294] INFO : Plotted number of elements +rank0 [2026-10-29 01:04:07,848] INFO : Plotted jets +rank0 [2026-10-29 01:04:08,990] INFO : Plotted jet ratio +rank0 [2026-10-29 01:04:09,041] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits +rank0 [2026-10-29 01:04:11,556] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits +rank0 [2026-10-29 01:04:12,418] INFO : Plotted number of elements +rank0 [2026-10-29 01:04:14,377] INFO : Plotted jets +rank0 [2026-10-29 01:04:15,528] INFO : Plotted jet ratio +rank0 [2026-10-29 01:09:15,083] INFO : Step 16100/50000 rank0 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2365 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0794 | Regression_eta: 0.0528 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0709 | LR: 7.89e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.8826s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank0 [2026-10-29 01:09:15,338] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:14:08,802] INFO : Step 16200/50000 rank0 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1962 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0871 | Regression_eta: 0.0440 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0963 | LR: 7.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8415s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank0 [2026-10-29 01:14:09,042] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:19:04,548] INFO : Step 16300/50000 rank0 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0861 | Regression_eta: 0.0675 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0759 | LR: 7.84e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.8666s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank0 [2026-10-29 01:19:04,788] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:23:56,874] INFO : Step 16400/50000 rank0 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0730 | Regression_eta: 0.0562 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0634 | LR: 7.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3744s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank0 [2026-10-29 01:23:57,114] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:28:52,365] INFO : Step 16500/50000 rank0 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0794 | Regression_eta: 0.0472 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0812 | LR: 7.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.0052s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank0 [2026-10-29 01:28:52,608] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:33:47,050] INFO : Step 16600/50000 rank0 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0780 | Regression_eta: 0.0501 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0647 | LR: 7.77e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.0792s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank0 [2026-10-29 01:33:47,328] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 53.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:38:39,327] INFO : Step 16700/50000 rank0 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2308 | Regression_cos_phi: 0.1252 | Regression_energy: 0.0953 | Regression_eta: 0.0870 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0892 | LR: 7.74e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.6184s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank0 [2026-10-29 01:38:39,587] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:43:31,638] INFO : Step 16800/50000 rank0 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.2180 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0812 | Regression_eta: 0.0550 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0838 | LR: 7.72e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.3280s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank0 [2026-10-29 01:43:31,878] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:48:24,338] INFO : Step 16900/50000 rank0 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0662 | Regression_eta: 0.0481 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0892 | LR: 7.69e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0304s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank0 [2026-10-29 01:48:24,603] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:53:20,321] INFO : Step 17000/50000 rank0 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2308 | Regression_cos_phi: 0.1029 | Regression_energy: 0.0778 | Regression_eta: 0.0765 | Regression_pt: 0.0816 | Regression_sin_phi: 0.1357 | LR: 7.67e-04 | DataLoad Time: 0.0214s | Model Forward Time: 6.4340s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank0 [2026-10-29 01:53:20,560] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:53:20,752] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-17000.pth +rank0 [2026-10-29 01:53:26,908] INFO : Running validation on rank0 +rank0 [2026-10-29 01:53:26,908] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 01:53:26,908] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION | Step=17000/50000 | Train Loss=1.2176 | Valid Loss=1.1828 | Train Loss Unweighted=0.6165 | Valid Loss Unweighted=0.5975 | Stale=0 | ETA=1710.4m +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 01:54:27,358] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari=0.528789 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.616193 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_competing=0.688842 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.624705 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0430621 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0939007 +rank0 [2026-10-29 01:54:27,359] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/seconds_per_rank=21.9736 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 01:54:27,360] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari=0.294766 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding_minus_raw/ari=0.164326 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 01:54:27,361] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 01:54:27,367] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:54:27,639] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 01:54:27,643] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 01:55:04,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 01:55:05,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 01:55:07,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 01:55:08,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 01:55:18,326] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 01:55:18,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:55:18,855] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 01:55:18,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:55:34,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 01:55:36,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 01:55:37,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 01:55:39,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 01:55:49,101] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:55:50,413] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 01:55:50,418] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 01:56:06,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 01:56:07,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 01:56:08,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 01:56:09,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 01:56:19,115] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 01:56:19,173] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:56:22,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits +rank0 [2026-10-29 01:56:23,499] INFO : Plotted number of elements +rank0 [2026-10-29 01:56:25,409] INFO : Plotted jets +rank0 [2026-10-29 01:56:26,543] INFO : Plotted jet ratio +rank0 [2026-10-29 01:56:26,631] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:56:29,877] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 01:56:30,849] INFO : Plotted number of elements +rank0 [2026-10-29 01:56:32,352] INFO : Plotted jets +rank0 [2026-10-29 01:56:33,519] INFO : Plotted jet ratio +rank0 [2026-10-29 01:56:33,571] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits +rank0 [2026-10-29 01:56:36,050] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits +rank0 [2026-10-29 01:56:36,877] INFO : Plotted number of elements +rank0 [2026-10-29 01:56:38,378] INFO : Plotted jets +rank0 [2026-10-29 01:56:39,517] INFO : Plotted jet ratio +rank0 [2026-10-29 02:01:32,924] INFO : Step 17100/50000 rank0 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2703 | Regression_cos_phi: 0.1106 | Regression_energy: 0.0819 | Regression_eta: 0.0610 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0948 | LR: 7.64e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.7061s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank0 [2026-10-29 02:01:33,176] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 53.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:06:25,121] INFO : Step 17200/50000 rank0 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0760 | Regression_eta: 0.0426 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0823 | LR: 7.62e-04 | DataLoad Time: 0.0182s | Model Forward Time: 6.2239s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank0 [2026-10-29 02:06:25,361] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 0% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 44% 2% +4 8 0x7408, 13395 53.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 93% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:11:07,261] INFO : Step 17300/50000 rank0 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0689 | Regression_eta: 0.0402 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0665 | LR: 7.59e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.5050s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank0 [2026-10-29 02:11:07,505] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 5% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:16:08,179] INFO : Step 17400/50000 rank0 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2405 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0763 | Regression_eta: 0.0626 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0932 | LR: 7.57e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2044s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank0 [2026-10-29 02:16:08,441] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:21:02,805] INFO : Step 17500/50000 rank0 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0774 | Regression_eta: 0.0361 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0657 | LR: 7.54e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.9508s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank0 [2026-10-29 02:21:03,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:26:00,100] INFO : Step 17600/50000 rank0 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0626 | Regression_eta: 0.0464 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0588 | LR: 7.52e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.2605s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank0 [2026-10-29 02:26:00,354] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:30:47,682] INFO : Step 17700/50000 rank0 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0663 | Regression_energy: 0.1031 | Regression_eta: 0.0559 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0712 | LR: 7.49e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.4000s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank0 [2026-10-29 02:30:47,941] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:35:38,423] INFO : Step 17800/50000 rank0 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0706 | Regression_eta: 0.0547 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0751 | LR: 7.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.7477s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank0 [2026-10-29 02:35:38,663] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:40:31,506] INFO : Step 17900/50000 rank0 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0685 | Regression_eta: 0.0365 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0616 | LR: 7.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3526s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank0 [2026-10-29 02:40:31,750] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:45:16,947] INFO : Step 18000/50000 rank0 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0740 | Regression_eta: 0.0529 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0713 | LR: 7.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5137s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank0 [2026-10-29 02:45:17,188] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:45:17,210] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-18000.pth +rank0 [2026-10-29 02:45:17,618] INFO : Running validation on rank0 +rank0 [2026-10-29 02:45:17,618] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:45:17,618] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:46:13,300] INFO : VALIDATION | Step=18000/50000 | Train Loss=1.1292 | Valid Loss=1.1094 | Train Loss Unweighted=0.5474 | Valid Loss Unweighted=0.5638 | Stale=0 | ETA=1658.4m +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 02:46:13,301] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari=0.530359 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.66754 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_competing=0.752217 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.638216 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0446315 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.107411 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 02:46:13,302] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/seconds_per_rank=22.0805 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari=0.296694 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding_minus_raw/ari=0.166254 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 02:46:13,303] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 02:46:13,304] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:46:13,308] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:46:13,568] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:46:13,573] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 02:46:39,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:46:40,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:46:41,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:46:43,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:46:53,083] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:46:53,115] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:46:53,608] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:46:53,613] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:47:09,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:47:11,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:47:13,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:47:15,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:47:25,301] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 02:47:25,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:47:25,821] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 02:47:25,824] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 02:47:41,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:47:42,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:47:43,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:47:44,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:47:54,622] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 02:47:54,804] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:47:58,105] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits +rank0 [2026-10-29 02:47:59,213] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:00,826] INFO : Plotted jets +rank0 [2026-10-29 02:48:02,025] INFO : Plotted jet ratio +rank0 [2026-10-29 02:48:02,111] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:48:05,712] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 02:48:06,747] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:08,249] INFO : Plotted jets +rank0 [2026-10-29 02:48:09,426] INFO : Plotted jet ratio +rank0 [2026-10-29 02:48:09,475] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits +rank0 [2026-10-29 02:48:11,900] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits +rank0 [2026-10-29 02:48:12,799] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:14,308] INFO : Plotted jets +rank0 [2026-10-29 02:48:15,488] INFO : Plotted jet ratio +rank0 [2026-10-29 02:53:15,119] INFO : Step 18100/50000 rank0 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0619 | Regression_eta: 0.0572 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0472 | LR: 7.39e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.3669s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank0 [2026-10-29 02:53:15,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 98% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:58:09,272] INFO : Step 18200/50000 rank0 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0746 | Regression_eta: 0.0389 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0822 | LR: 7.36e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.5920s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank0 [2026-10-29 02:58:09,513] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:02:55,799] INFO : Step 18300/50000 rank0 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0745 | Regression_eta: 0.0406 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0542 | LR: 7.34e-04 | DataLoad Time: 0.0216s | Model Forward Time: 6.3257s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank0 [2026-10-29 03:02:56,062] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 54.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:07:46,735] INFO : Step 18400/50000 rank0 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2135 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0759 | Regression_eta: 0.0705 | Regression_pt: 0.0783 | Regression_sin_phi: 0.1128 | LR: 7.31e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.4716s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank0 [2026-10-29 03:07:46,988] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:12:44,242] INFO : Step 18500/50000 rank0 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2224 | Regression_cos_phi: 0.1209 | Regression_energy: 0.0732 | Regression_eta: 0.0654 | Regression_pt: 0.0752 | Regression_sin_phi: 0.1021 | LR: 7.29e-04 | DataLoad Time: 0.0128s | Model Forward Time: 6.0729s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank0 [2026-10-29 03:12:44,485] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:17:40,213] INFO : Step 18600/50000 rank0 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0588 | Regression_energy: 0.1103 | Regression_eta: 0.0535 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0679 | LR: 7.26e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.2194s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank0 [2026-10-29 03:17:40,455] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:22:31,843] INFO : Step 18700/50000 rank0 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0681 | Regression_eta: 0.0424 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0540 | LR: 7.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1560s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank0 [2026-10-29 03:22:32,105] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:27:38,022] INFO : Step 18800/50000 rank0 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0942 | Regression_eta: 0.0561 | Regression_pt: 0.0971 | Regression_sin_phi: 0.1156 | LR: 7.21e-04 | DataLoad Time: 0.0169s | Model Forward Time: 7.7298s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank0 [2026-10-29 03:27:38,264] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 53.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:32:27,593] INFO : Step 18900/50000 rank0 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0738 | Regression_eta: 0.0404 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0496 | LR: 7.18e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1849s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank0 [2026-10-29 03:32:27,832] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 55.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:37:07,387] INFO : Step 19000/50000 rank0 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0770 | Regression_eta: 0.0605 | Regression_pt: 0.0780 | Regression_sin_phi: 0.1183 | LR: 7.16e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.3469s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank0 [2026-10-29 03:37:07,632] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:37:07,735] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-19000.pth +rank0 [2026-10-29 03:37:08,025] INFO : Running validation on rank0 +rank0 [2026-10-29 03:37:08,026] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 03:37:08,026] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION | Step=19000/50000 | Train Loss=1.1912 | Valid Loss=1.1240 | Train Loss Unweighted=0.5756 | Valid Loss Unweighted=0.5605 | Stale=1 | ETA=1606.6m +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 03:38:00,820] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari=0.543106 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.680915 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_competing=0.769869 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.620868 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0573789 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0900633 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 03:38:00,821] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/seconds_per_rank=22.1725 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:38:00,822] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari=0.302827 +rank0 [2026-10-29 03:38:00,823] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 03:38:00,823] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding_minus_raw/ari=0.172387 +rank0 [2026-10-29 03:38:00,823] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 03:38:00,823] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 03:38:00,823] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 03:38:00,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:38:01,080] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 03:38:01,087] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 03:38:27,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 03:38:28,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 03:38:29,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 03:38:30,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 03:38:40,757] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:38:40,770] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:38:41,427] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 03:38:41,431] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:38:57,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 03:38:58,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 03:38:59,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 03:39:00,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 03:39:10,569] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:39:11,226] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 03:39:11,230] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 03:39:27,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 03:39:28,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 03:39:29,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 03:39:30,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 03:39:40,141] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 03:39:40,157] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits +rank0 [2026-10-29 03:39:43,562] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits +rank0 [2026-10-29 03:39:44,669] INFO : Plotted number of elements +rank0 [2026-10-29 03:39:46,290] INFO : Plotted jets +rank0 [2026-10-29 03:39:47,474] INFO : Plotted jet ratio +rank0 [2026-10-29 03:39:47,555] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:39:50,851] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 03:39:51,903] INFO : Plotted number of elements +rank0 [2026-10-29 03:39:53,462] INFO : Plotted jets +rank0 [2026-10-29 03:39:54,601] INFO : Plotted jet ratio +rank0 [2026-10-29 03:39:54,652] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits +rank0 [2026-10-29 03:39:57,142] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits +rank0 [2026-10-29 03:39:57,991] INFO : Plotted number of elements +rank0 [2026-10-29 03:40:00,033] INFO : Plotted jets +rank0 [2026-10-29 03:40:01,204] INFO : Plotted jet ratio +rank0 [2026-10-29 03:44:43,407] INFO : Step 19100/50000 rank0 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2506 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0861 | Regression_eta: 0.0631 | Regression_pt: 0.0900 | Regression_sin_phi: 0.1165 | LR: 7.13e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6259s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank0 [2026-10-29 03:44:43,660] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:49:24,238] INFO : Step 19200/50000 rank0 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0645 | Regression_eta: 0.0315 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0694 | LR: 7.10e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2503s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank0 [2026-10-29 03:49:24,491] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:54:09,869] INFO : Step 19300/50000 rank0 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0637 | Regression_eta: 0.0372 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0425 | LR: 7.08e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1060s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank0 [2026-10-29 03:54:10,114] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:58:51,980] INFO : Step 19400/50000 rank0 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0839 | Regression_eta: 0.0529 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0553 | LR: 7.05e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.3796s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank0 [2026-10-29 03:58:52,223] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:03:31,342] INFO : Step 19500/50000 rank0 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0659 | Regression_eta: 0.0521 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0666 | LR: 7.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4150s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank0 [2026-10-29 04:03:31,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:08:13,211] INFO : Step 19600/50000 rank0 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0671 | Regression_eta: 0.0585 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0634 | LR: 7.00e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.1632s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank0 [2026-10-29 04:08:13,454] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:12:52,984] INFO : Step 19700/50000 rank0 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1883 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0744 | Regression_eta: 0.0576 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0735 | LR: 6.97e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9933s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank0 [2026-10-29 04:12:53,226] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 183.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:17:29,078] INFO : Step 19800/50000 rank0 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0606 | Regression_eta: 0.0423 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0637 | LR: 6.94e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8090s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank0 [2026-10-29 04:17:29,318] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:22:09,638] INFO : Step 19900/50000 rank0 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2194 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0870 | Regression_eta: 0.0598 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0817 | LR: 6.92e-04 | DataLoad Time: 0.0182s | Model Forward Time: 6.0747s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank0 [2026-10-29 04:22:09,883] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 8% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 98% +2 6 0x7408, 15961 47.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 99% +4 8 0x7408, 13395 54.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:26:52,705] INFO : Step 20000/50000 rank0 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0611 | Regression_eta: 0.0434 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0683 | LR: 6.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3873s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank0 [2026-10-29 04:26:52,968] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:26:52,990] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-20000.pth +rank0 [2026-10-29 04:26:53,279] INFO : Running validation on rank0 +rank0 [2026-10-29 04:26:53,279] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:26:53,279] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:27:46,550] INFO : VALIDATION | Step=20000/50000 | Train Loss=1.0534 | Valid Loss=1.0747 | Train Loss Unweighted=0.5290 | Valid Loss Unweighted=0.5440 | Stale=0 | ETA=1551.6m +rank0 [2026-10-29 04:27:46,550] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:27:46,550] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:27:46,550] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari=0.557767 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.672076 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_competing=0.759574 +rank0 [2026-10-29 04:27:46,551] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.646746 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0720397 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.115942 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 04:27:46,552] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/seconds_per_rank=22.1758 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari=0.282324 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari_threshold=0.84375 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding_minus_raw/ari=0.151884 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 04:27:46,553] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:27:46,559] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:27:46,816] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:27:46,822] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 04:28:14,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:28:15,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:28:16,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:28:17,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:28:27,941] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:28:28,500] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:28:28,504] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:28:44,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:28:45,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:28:47,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:28:48,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:28:58,685] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 04:28:58,727] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:28:59,251] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 04:28:59,254] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 04:29:15,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:29:15,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:29:16,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:29:17,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:29:27,959] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 04:29:27,976] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:29:31,696] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits +rank0 [2026-10-29 04:29:32,807] INFO : Plotted number of elements +rank0 [2026-10-29 04:29:34,360] INFO : Plotted jets +rank0 [2026-10-29 04:29:35,500] INFO : Plotted jet ratio +rank0 [2026-10-29 04:29:35,567] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:29:38,833] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 04:29:39,830] INFO : Plotted number of elements +rank0 [2026-10-29 04:29:41,353] INFO : Plotted jets +rank0 [2026-10-29 04:29:42,477] INFO : Plotted jet ratio +rank0 [2026-10-29 04:29:42,532] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits +rank0 [2026-10-29 04:29:45,042] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits +rank0 [2026-10-29 04:29:45,893] INFO : Plotted number of elements +rank0 [2026-10-29 04:29:47,399] INFO : Plotted jets +rank0 [2026-10-29 04:29:49,093] INFO : Plotted jet ratio +rank0 [2026-10-29 04:34:27,739] INFO : Step 20100/50000 rank0 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2137 | Regression_cos_phi: 0.1007 | Regression_energy: 0.0740 | Regression_eta: 0.0482 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0856 | LR: 6.86e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.9064s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank0 [2026-10-29 04:34:28,000] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:39:09,211] INFO : Step 20200/50000 rank0 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0600 | Regression_eta: 0.0402 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0552 | LR: 6.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.6006s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank0 [2026-10-29 04:39:09,477] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 97% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 96% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:43:47,800] INFO : Step 20300/50000 rank0 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0486 | Regression_eta: 0.0371 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0651 | LR: 6.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1921s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank0 [2026-10-29 04:43:48,092] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:48:27,683] INFO : Step 20400/50000 rank0 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0853 | Regression_eta: 0.0370 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0540 | LR: 6.78e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3137s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank0 [2026-10-29 04:48:27,952] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:53:09,097] INFO : Step 20500/50000 rank0 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0748 | Regression_eta: 0.0487 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0672 | LR: 6.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.6942s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank0 [2026-10-29 04:53:09,361] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:57:54,174] INFO : Step 20600/50000 rank0 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0728 | Regression_eta: 0.0373 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0594 | LR: 6.73e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6302s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank0 [2026-10-29 04:57:54,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 5% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 90% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:02:48,559] INFO : Step 20700/50000 rank0 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0690 | Regression_eta: 0.0540 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0743 | LR: 6.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.3068s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank0 [2026-10-29 05:02:48,819] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:07:35,442] INFO : Step 20800/50000 rank0 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.2376 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0716 | Regression_eta: 0.0488 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0606 | LR: 6.67e-04 | DataLoad Time: 0.0196s | Model Forward Time: 7.6364s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank0 [2026-10-29 05:07:35,684] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:12:19,167] INFO : Step 20900/50000 rank0 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0667 | Regression_eta: 0.0340 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0578 | LR: 6.65e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2957s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank0 [2026-10-29 05:12:19,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:17:04,642] INFO : Step 21000/50000 rank0 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0634 | Regression_eta: 0.0436 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0542 | LR: 6.62e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2368s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank0 [2026-10-29 05:17:04,882] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:17:04,907] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-21000.pth +rank0 [2026-10-29 05:17:05,306] INFO : Running validation on rank0 +rank0 [2026-10-29 05:17:05,306] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 05:17:05,306] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 05:17:58,968] INFO : VALIDATION | Step=21000/50000 | Train Loss=1.0251 | Valid Loss=1.0712 | Train Loss Unweighted=0.4983 | Valid Loss Unweighted=0.5337 | Stale=0 | ETA=1497.8m +rank0 [2026-10-29 05:17:58,968] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 05:17:58,969] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari=0.525349 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.660363 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_competing=0.743627 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.637515 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.039622 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.10671 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 05:17:58,970] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/seconds_per_rank=21.8211 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari=0.311715 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding_minus_raw/ari=0.181275 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 05:17:58,971] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 05:17:58,972] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 05:17:58,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:17:59,233] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 05:17:59,238] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 05:18:22,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 05:18:24,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 05:18:25,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 05:18:26,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 05:18:36,550] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:18:37,117] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 05:18:37,120] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:18:53,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 05:18:54,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 05:18:55,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 05:18:56,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 05:19:06,691] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 05:19:06,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:19:07,186] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 05:19:07,189] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 05:19:22,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 05:19:23,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 05:19:24,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 05:19:25,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 05:19:35,466] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 05:19:35,762] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits +rank0 [2026-10-29 05:19:38,982] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits +rank0 [2026-10-29 05:19:40,053] INFO : Plotted number of elements +rank0 [2026-10-29 05:19:41,993] INFO : Plotted jets +rank0 [2026-10-29 05:19:43,128] INFO : Plotted jet ratio +rank0 [2026-10-29 05:19:43,199] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:19:46,493] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 05:19:47,493] INFO : Plotted number of elements +rank0 [2026-10-29 05:19:48,996] INFO : Plotted jets +rank0 [2026-10-29 05:19:50,159] INFO : Plotted jet ratio +rank0 [2026-10-29 05:19:50,217] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits +rank0 [2026-10-29 05:19:52,752] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits +rank0 [2026-10-29 05:19:53,655] INFO : Plotted number of elements +rank0 [2026-10-29 05:19:55,236] INFO : Plotted jets +rank0 [2026-10-29 05:19:56,406] INFO : Plotted jet ratio +rank0 [2026-10-29 05:24:39,297] INFO : Step 21100/50000 rank0 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0647 | Regression_eta: 0.0550 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0619 | LR: 6.59e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.0978s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank0 [2026-10-29 05:24:39,578] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 5% +4 8 0x7408, 13395 54.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 05:29:25,365] INFO : Step 21200/50000 rank0 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0793 | Regression_eta: 0.0557 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0495 | LR: 6.56e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.4850s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank0 [2026-10-29 05:29:25,604] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:34:39,767] INFO : Step 21300/50000 rank0 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0639 | Regression_eta: 0.0544 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0589 | LR: 6.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3240s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank0 [2026-10-29 05:34:40,026] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:39:30,466] INFO : Step 21400/50000 rank0 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0978 | Regression_energy: 0.0708 | Regression_eta: 0.0703 | Regression_pt: 0.0731 | Regression_sin_phi: 0.1031 | LR: 6.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.1716s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank0 [2026-10-29 05:39:30,708] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:44:18,446] INFO : Step 21500/50000 rank0 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1403 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0785 | Regression_eta: 0.0515 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0575 | LR: 6.48e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.0169s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank0 [2026-10-29 05:44:18,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:49:11,825] INFO : Step 21600/50000 rank0 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0639 | Regression_eta: 0.0459 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0781 | LR: 6.45e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3005s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank0 [2026-10-29 05:49:12,070] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:54:00,361] INFO : Step 21700/50000 rank0 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2473 | Regression_cos_phi: 0.1118 | Regression_energy: 0.0745 | Regression_eta: 0.0721 | Regression_pt: 0.0750 | Regression_sin_phi: 0.1078 | LR: 6.43e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.3487s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank0 [2026-10-29 05:54:00,602] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:58:53,223] INFO : Step 21800/50000 rank0 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0656 | Regression_eta: 0.0294 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0645 | LR: 6.40e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9130s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank0 [2026-10-29 05:58:53,466] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:03:50,088] INFO : Step 21900/50000 rank0 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0977 | Regression_eta: 0.0511 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0614 | LR: 6.37e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5204s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank0 [2026-10-29 06:03:50,340] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:08:51,459] INFO : Step 22000/50000 rank0 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2182 | Regression_cos_phi: 0.1124 | Regression_energy: 0.0796 | Regression_eta: 0.0606 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0777 | LR: 6.34e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.2208s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank0 [2026-10-29 06:08:51,701] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 98% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:08:51,825] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-22000.pth +rank0 [2026-10-29 06:08:52,159] INFO : Running validation on rank0 +rank0 [2026-10-29 06:08:52,159] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 06:08:52,159] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION | Step=22000/50000 | Train Loss=1.2256 | Valid Loss=1.1094 | Train Loss Unweighted=0.5955 | Valid Loss Unweighted=0.5599 | Stale=1 | ETA=1446.4m +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 06:09:46,271] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari=0.536563 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.646099 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_competing=0.727607 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.642788 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0508356 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.111983 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 06:09:46,272] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/seconds_per_rank=22.1163 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 06:09:46,273] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari=0.305526 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding_minus_raw/ari=0.175086 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 06:09:46,274] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 06:09:46,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:09:46,561] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 06:09:46,565] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 06:10:12,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 06:10:13,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 06:10:15,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 06:10:16,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 06:10:26,861] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 06:10:26,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:10:27,422] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 06:10:27,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:10:43,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 06:10:44,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 06:10:45,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 06:10:46,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 06:10:56,833] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 06:10:56,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:10:57,446] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 06:10:57,450] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 06:11:13,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 06:11:14,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 06:11:15,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 06:11:16,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 06:11:26,428] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 06:11:26,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits +rank0 [2026-10-29 06:11:29,741] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits +rank0 [2026-10-29 06:11:30,858] INFO : Plotted number of elements +rank0 [2026-10-29 06:11:32,422] INFO : Plotted jets +rank0 [2026-10-29 06:11:33,594] INFO : Plotted jet ratio +rank0 [2026-10-29 06:11:33,644] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:11:36,955] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 06:11:37,999] INFO : Plotted number of elements +rank0 [2026-10-29 06:11:40,102] INFO : Plotted jets +rank0 [2026-10-29 06:11:41,290] INFO : Plotted jet ratio +rank0 [2026-10-29 06:11:41,340] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits +rank0 [2026-10-29 06:11:43,793] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits +rank0 [2026-10-29 06:11:44,655] INFO : Plotted number of elements +rank0 [2026-10-29 06:11:46,195] INFO : Plotted jets +rank0 [2026-10-29 06:11:47,391] INFO : Plotted jet ratio +rank0 [2026-10-29 06:16:43,527] INFO : Step 22100/50000 rank0 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0659 | Regression_eta: 0.0485 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0767 | LR: 6.32e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.8325s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank0 [2026-10-29 06:16:43,784] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:21:32,707] INFO : Step 22200/50000 rank0 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0706 | Regression_eta: 0.0739 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0831 | LR: 6.29e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.2797s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank0 [2026-10-29 06:21:32,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:26:34,046] INFO : Step 22300/50000 rank0 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0535 | Regression_eta: 0.0438 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0592 | LR: 6.26e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.3846s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank0 [2026-10-29 06:26:34,289] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:31:30,679] INFO : Step 22400/50000 rank0 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0757 | Regression_eta: 0.0445 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0644 | LR: 6.23e-04 | DataLoad Time: 0.0134s | Model Forward Time: 7.2017s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank0 [2026-10-29 06:31:30,941] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 53.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:36:32,164] INFO : Step 22500/50000 rank0 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0581 | Regression_eta: 0.0261 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0637 | LR: 6.20e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.9045s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank0 [2026-10-29 06:36:32,404] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:41:28,851] INFO : Step 22600/50000 rank0 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0730 | Regression_eta: 0.0699 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0681 | LR: 6.18e-04 | DataLoad Time: 0.0251s | Model Forward Time: 5.0911s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank0 [2026-10-29 06:41:29,100] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 47.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 98% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 4% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 97% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:46:29,890] INFO : Step 22700/50000 rank0 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0649 | Regression_eta: 0.0458 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0792 | LR: 6.15e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.6388s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank0 [2026-10-29 06:46:30,133] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 47.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:51:27,596] INFO : Step 22800/50000 rank0 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0604 | Regression_eta: 0.0392 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0571 | LR: 6.12e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.1799s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank0 [2026-10-29 06:51:27,843] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 1% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:56:23,980] INFO : Step 22900/50000 rank0 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0645 | Regression_eta: 0.0293 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0458 | LR: 6.09e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.4366s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank0 [2026-10-29 06:56:24,223] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:01:33,846] INFO : Step 23000/50000 rank0 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.2235 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0666 | Regression_eta: 0.0624 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0795 | LR: 6.06e-04 | DataLoad Time: 0.0186s | Model Forward Time: 6.2591s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank0 [2026-10-29 07:01:34,089] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 53.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:01:34,114] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-23000.pth +rank0 [2026-10-29 07:01:34,582] INFO : Running validation on rank0 +rank0 [2026-10-29 07:01:34,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:01:34,583] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:02:29,119] INFO : VALIDATION | Step=23000/50000 | Train Loss=1.1236 | Valid Loss=1.0549 | Train Loss Unweighted=0.5411 | Valid Loss Unweighted=0.5211 | Stale=0 | ETA=1395.9m +rank0 [2026-10-29 07:02:29,119] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:02:29,119] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 07:02:29,120] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari=0.549497 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.671263 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_competing=0.759502 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.650956 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0637701 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.120151 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:02:29,121] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/seconds_per_rank=21.9469 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:02:29,122] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari=0.29218 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding_minus_raw/ari=0.16174 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 07:02:29,123] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:02:29,128] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:02:29,389] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:02:29,394] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 07:02:49,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:02:50,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:02:52,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:02:53,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:03:03,337] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:03:03,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:03:04,118] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:03:04,121] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:03:20,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:03:21,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:03:22,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:03:23,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:03:33,875] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 07:03:33,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:03:34,182] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 07:03:34,185] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 07:03:49,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:03:50,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:03:51,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:03:52,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:04:02,903] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 07:04:02,921] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:04:06,247] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:04:07,340] INFO : Plotted number of elements +rank0 [2026-10-29 07:04:08,909] INFO : Plotted jets +rank0 [2026-10-29 07:04:10,038] INFO : Plotted jet ratio +rank0 [2026-10-29 07:04:10,092] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:04:13,406] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:04:14,472] INFO : Plotted number of elements +rank0 [2026-10-29 07:04:15,992] INFO : Plotted jets +rank0 [2026-10-29 07:04:17,101] INFO : Plotted jet ratio +rank0 [2026-10-29 07:04:17,158] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits +rank0 [2026-10-29 07:04:19,643] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits +rank0 [2026-10-29 07:04:20,518] INFO : Plotted number of elements +rank0 [2026-10-29 07:04:22,526] INFO : Plotted jets +rank0 [2026-10-29 07:04:23,688] INFO : Plotted jet ratio +rank0 [2026-10-29 07:09:14,671] INFO : Step 23100/50000 rank0 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0730 | Regression_eta: 0.0689 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0610 | LR: 6.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.8794s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank0 [2026-10-29 07:09:14,916] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:14:13,618] INFO : Step 23200/50000 rank0 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0668 | Regression_eta: 0.0501 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0549 | LR: 6.01e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.5104s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank0 [2026-10-29 07:14:13,861] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:19:06,482] INFO : Step 23300/50000 rank0 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0670 | Regression_eta: 0.0505 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0877 | LR: 5.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5420s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank0 [2026-10-29 07:19:06,724] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:24:01,994] INFO : Step 23400/50000 rank0 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1768 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0569 | Regression_eta: 0.0395 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0494 | LR: 5.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.8716s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank0 [2026-10-29 07:24:02,239] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 1% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 49% 0% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 14% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:29:15,327] INFO : Step 23500/50000 rank0 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0652 | Regression_eta: 0.0392 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0723 | LR: 5.92e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.4485s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank0 [2026-10-29 07:29:15,568] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:34:23,013] INFO : Step 23600/50000 rank0 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0726 | Regression_eta: 0.0310 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0551 | LR: 5.90e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.0868s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank0 [2026-10-29 07:34:23,257] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:39:42,362] INFO : Step 23700/50000 rank0 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0602 | Regression_eta: 0.0559 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0833 | LR: 5.87e-04 | DataLoad Time: 0.0180s | Model Forward Time: 7.2851s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank0 [2026-10-29 07:39:42,604] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 46.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:44:47,313] INFO : Step 23800/50000 rank0 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0707 | Regression_eta: 0.0546 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0650 | LR: 5.84e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.5076s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank0 [2026-10-29 07:44:47,559] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:49:50,683] INFO : Step 23900/50000 rank0 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0708 | Regression_eta: 0.0371 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0645 | LR: 5.81e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.5792s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank0 [2026-10-29 07:49:50,928] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:54:59,586] INFO : Step 24000/50000 rank0 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0590 | Regression_eta: 0.0389 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0846 | LR: 5.78e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5440s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank0 [2026-10-29 07:54:59,830] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:54:59,855] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-24000.pth +rank0 [2026-10-29 07:55:00,243] INFO : Running validation on rank0 +rank0 [2026-10-29 07:55:00,243] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:55:00,243] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:55:54,253] INFO : VALIDATION | Step=24000/50000 | Train Loss=1.0214 | Valid Loss=1.0099 | Train Loss Unweighted=0.4973 | Valid Loss Unweighted=0.4976 | Stale=0 | ETA=1346.1m +rank0 [2026-10-29 07:55:54,253] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 07:55:54,254] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari=0.565644 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.685175 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_competing=0.773691 +rank0 [2026-10-29 07:55:54,255] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.644545 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0799171 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.11374 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 07:55:54,256] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/seconds_per_rank=21.9234 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:55:54,257] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari=0.283118 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding_minus_raw/ari=0.152678 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 07:55:54,258] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:55:54,262] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:55:54,530] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:55:54,537] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 07:56:20,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:56:22,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:56:23,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:56:24,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:56:34,467] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:56:34,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:56:35,051] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:56:35,054] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:56:51,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:56:52,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:56:54,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:56:55,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:57:05,105] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:57:05,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:57:05,693] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 07:57:05,697] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 07:57:21,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:57:23,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:57:23,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:57:24,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:57:34,973] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 07:57:34,989] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:57:38,288] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits +rank0 [2026-10-29 07:57:39,388] INFO : Plotted number of elements +rank0 [2026-10-29 07:57:41,373] INFO : Plotted jets +rank0 [2026-10-29 07:57:42,530] INFO : Plotted jet ratio +rank0 [2026-10-29 07:57:42,606] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:57:45,997] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 07:57:46,996] INFO : Plotted number of elements +rank0 [2026-10-29 07:57:48,510] INFO : Plotted jets +rank0 [2026-10-29 07:57:49,629] INFO : Plotted jet ratio +rank0 [2026-10-29 07:57:49,683] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits +rank0 [2026-10-29 07:57:52,238] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits +rank0 [2026-10-29 07:57:53,107] INFO : Plotted number of elements +rank0 [2026-10-29 07:57:54,650] INFO : Plotted jets +rank0 [2026-10-29 07:57:55,861] INFO : Plotted jet ratio +rank0 [2026-10-29 08:03:05,386] INFO : Step 24100/50000 rank0 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2256 | Regression_cos_phi: 0.1024 | Regression_energy: 0.0585 | Regression_eta: 0.0587 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0827 | LR: 5.75e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.3199s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank0 [2026-10-29 08:03:05,629] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 35% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 54.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 29% 0% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:08:00,736] INFO : Step 24200/50000 rank0 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0805 | Regression_eta: 0.0376 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0510 | LR: 5.73e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.0351s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank0 [2026-10-29 08:08:00,982] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 21% +4 8 0x7408, 13395 54.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 92% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:13:19,290] INFO : Step 24300/50000 rank0 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0533 | Regression_eta: 0.0433 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0601 | LR: 5.70e-04 | DataLoad Time: 0.0209s | Model Forward Time: 8.2603s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank0 [2026-10-29 08:13:19,533] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 98% +2 6 0x7408, 15961 47.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 21% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 93% +4 8 0x7408, 13395 52.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 2% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +6 10 0x7408, 62036 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:18:16,657] INFO : Step 24400/50000 rank0 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0741 | Regression_eta: 0.0523 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0506 | LR: 5.67e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.0180s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank0 [2026-10-29 08:18:16,901] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:23:17,407] INFO : Step 24500/50000 rank0 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0719 | Regression_eta: 0.0347 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0599 | LR: 5.64e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.2606s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank0 [2026-10-29 08:23:17,659] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 3% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 3% +2 6 0x7408, 15961 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 98% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:28:19,230] INFO : Step 24600/50000 rank0 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0602 | Regression_eta: 0.0498 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0678 | LR: 5.61e-04 | DataLoad Time: 0.0129s | Model Forward Time: 5.4337s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank0 [2026-10-29 08:28:19,474] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 2% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 94% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:33:22,560] INFO : Step 24700/50000 rank0 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0704 | Regression_eta: 0.0736 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0777 | LR: 5.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.6134s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank0 [2026-10-29 08:33:22,807] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:38:20,231] INFO : Step 24800/50000 rank0 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0531 | Regression_eta: 0.0436 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0797 | LR: 5.56e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.8941s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank0 [2026-10-29 08:38:20,475] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 3% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:43:22,230] INFO : Step 24900/50000 rank0 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0591 | Regression_eta: 0.0372 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0608 | LR: 5.53e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1702s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank0 [2026-10-29 08:43:22,483] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:48:26,515] INFO : Step 25000/50000 rank0 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0692 | Regression_eta: 0.0452 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0613 | LR: 5.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.6255s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank0 [2026-10-29 08:48:26,769] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:48:26,797] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-25000.pth +rank0 [2026-10-29 08:48:27,086] INFO : Running validation on rank0 +rank0 [2026-10-29 08:48:27,086] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 08:48:27,086] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 08:49:21,348] INFO : VALIDATION | Step=25000/50000 | Train Loss=0.9796 | Valid Loss=1.0385 | Train Loss Unweighted=0.4650 | Valid Loss Unweighted=0.5100 | Stale=1 | ETA=1296.0m +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 08:49:21,349] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari=0.538387 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.652425 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_competing=0.733781 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.645278 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0526593 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.114473 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 08:49:21,350] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/seconds_per_rank=22.0527 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari=0.294182 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding_minus_raw/ari=0.163743 +rank0 [2026-10-29 08:49:21,351] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 08:49:21,352] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 08:49:21,352] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 08:49:21,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:49:21,615] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 08:49:21,623] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 08:49:51,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 08:49:53,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 08:49:54,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 08:49:56,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 08:50:06,202] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:50:06,728] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 08:50:06,732] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:50:22,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 08:50:24,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 08:50:25,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 08:50:26,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 08:50:36,390] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 08:50:36,403] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:50:36,987] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 08:50:36,992] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 08:50:52,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 08:50:53,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 08:50:54,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 08:50:56,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 08:51:06,051] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 08:51:06,113] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits +rank0 [2026-10-29 08:51:09,435] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits +rank0 [2026-10-29 08:51:10,539] INFO : Plotted number of elements +rank0 [2026-10-29 08:51:12,161] INFO : Plotted jets +rank0 [2026-10-29 08:51:13,343] INFO : Plotted jet ratio +rank0 [2026-10-29 08:51:13,414] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:51:16,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 08:51:17,780] INFO : Plotted number of elements +rank0 [2026-10-29 08:51:19,335] INFO : Plotted jets +rank0 [2026-10-29 08:51:20,558] INFO : Plotted jet ratio +rank0 [2026-10-29 08:51:20,622] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits +rank0 [2026-10-29 08:51:23,149] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits +rank0 [2026-10-29 08:51:24,037] INFO : Plotted number of elements +rank0 [2026-10-29 08:51:26,129] INFO : Plotted jets +rank0 [2026-10-29 08:51:27,297] INFO : Plotted jet ratio +rank0 [2026-10-29 08:56:19,242] INFO : Step 25100/50000 rank0 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0497 | Regression_eta: 0.0382 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0574 | LR: 5.47e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.0807s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank0 [2026-10-29 08:56:19,487] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:01:02,461] INFO : Step 25200/50000 rank0 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0518 | Regression_eta: 0.0443 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0452 | LR: 5.44e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.7447s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank0 [2026-10-29 09:01:02,706] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 87% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 2% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:05:51,345] INFO : Step 25300/50000 rank0 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0558 | Regression_eta: 0.0383 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0664 | LR: 5.42e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.6567s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank0 [2026-10-29 09:05:51,590] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:10:31,376] INFO : Step 25400/50000 rank0 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1998 | Regression_cos_phi: 0.1042 | Regression_energy: 0.0582 | Regression_eta: 0.0535 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0670 | LR: 5.39e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.9281s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank0 [2026-10-29 09:10:31,620] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:15:11,874] INFO : Step 25500/50000 rank0 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0640 | Regression_eta: 0.0333 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0510 | LR: 5.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1060s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank0 [2026-10-29 09:15:12,119] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:19:57,475] INFO : Step 25600/50000 rank0 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0801 | Regression_eta: 0.0596 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0788 | LR: 5.33e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.8055s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank0 [2026-10-29 09:19:57,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 1% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 47.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 54.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 86% +6 10 0x7408, 62036 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:24:34,598] INFO : Step 25700/50000 rank0 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0625 | Regression_eta: 0.0405 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0543 | LR: 5.30e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.9468s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank0 [2026-10-29 09:24:34,842] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:29:28,343] INFO : Step 25800/50000 rank0 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0896 | Regression_eta: 0.0434 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0570 | LR: 5.27e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.8842s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank0 [2026-10-29 09:29:28,584] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:34:20,139] INFO : Step 25900/50000 rank0 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0639 | Regression_eta: 0.0467 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0536 | LR: 5.25e-04 | DataLoad Time: 0.0246s | Model Forward Time: 5.5776s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank0 [2026-10-29 09:34:20,388] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:39:04,132] INFO : Step 26000/50000 rank0 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0513 | Regression_eta: 0.0438 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0635 | LR: 5.22e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.3886s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank0 [2026-10-29 09:39:04,375] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:39:04,404] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-26000.pth +rank0 [2026-10-29 09:39:04,829] INFO : Running validation on rank0 +rank0 [2026-10-29 09:39:04,829] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:39:04,829] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:39:56,012] INFO : VALIDATION | Step=26000/50000 | Train Loss=0.9516 | Valid Loss=0.9756 | Train Loss Unweighted=0.4769 | Valid Loss Unweighted=0.4796 | Stale=0 | ETA=1243.0m +rank0 [2026-10-29 09:39:56,012] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:39:56,012] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:39:56,012] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari=0.575185 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.687065 +rank0 [2026-10-29 09:39:56,013] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_competing=0.77637 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.639714 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0894577 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.108909 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 09:39:56,014] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/seconds_per_rank=22.0047 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari=0.319408 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding_minus_raw/ari=0.188968 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 09:39:56,015] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:39:56,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:39:56,289] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:39:56,294] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 09:40:23,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:40:24,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:40:25,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:40:26,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:40:37,017] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 09:40:37,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:40:37,506] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:40:37,510] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:40:54,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:40:56,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:40:57,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:40:58,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:41:08,833] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 09:41:08,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:44:09,550] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 09:44:09,599] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 09:44:25,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:44:26,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:44:27,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:44:28,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:44:38,265] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 09:44:38,687] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:44:41,935] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits +rank0 [2026-10-29 09:44:49,158] INFO : Plotted number of elements +rank0 [2026-10-29 09:44:50,706] INFO : Plotted jets +rank0 [2026-10-29 09:44:54,990] INFO : Plotted jet ratio +rank0 [2026-10-29 09:45:21,925] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:45:25,227] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 09:45:26,257] INFO : Plotted number of elements +rank0 [2026-10-29 09:45:27,760] INFO : Plotted jets +rank0 [2026-10-29 09:45:28,876] INFO : Plotted jet ratio +rank0 [2026-10-29 09:45:28,940] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits +rank0 [2026-10-29 09:45:31,485] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits +rank0 [2026-10-29 09:45:32,337] INFO : Plotted number of elements +rank0 [2026-10-29 09:45:33,848] INFO : Plotted jets +rank0 [2026-10-29 09:45:35,022] INFO : Plotted jet ratio +rank0 [2026-10-29 09:50:30,271] INFO : Step 26100/50000 rank0 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.2137 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0684 | Regression_eta: 0.0604 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0890 | LR: 5.19e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.8916s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank0 [2026-10-29 09:50:30,515] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:55:16,311] INFO : Step 26200/50000 rank0 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0674 | Regression_eta: 0.0300 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0457 | LR: 5.16e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.8515s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank0 [2026-10-29 09:55:16,555] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 45% 2% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:59:58,405] INFO : Step 26300/50000 rank0 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0673 | Regression_eta: 0.0352 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0744 | LR: 5.13e-04 | DataLoad Time: 0.0234s | Model Forward Time: 5.5903s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank0 [2026-10-29 09:59:58,657] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:04:37,175] INFO : Step 26400/50000 rank0 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0501 | Regression_eta: 0.0357 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0653 | LR: 5.10e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.8419s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank0 [2026-10-29 10:04:37,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 9% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +6 10 0x7408, 62036 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:09:24,466] INFO : Step 26500/50000 rank0 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0512 | Regression_eta: 0.0323 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0484 | LR: 5.08e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.8127s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank0 [2026-10-29 10:09:24,710] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 48.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 52% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:14:06,430] INFO : Step 26600/50000 rank0 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1091 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0659 | Regression_eta: 0.0342 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0494 | LR: 5.05e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.3105s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank0 [2026-10-29 10:14:06,671] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:19:02,991] INFO : Step 26700/50000 rank0 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1556 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0630 | Regression_eta: 0.0402 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0721 | LR: 5.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.9600s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank0 [2026-10-29 10:19:03,276] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:23:59,335] INFO : Step 26800/50000 rank0 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0619 | Regression_eta: 0.0429 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0851 | LR: 4.99e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1903s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank0 [2026-10-29 10:23:59,578] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:28:52,219] INFO : Step 26900/50000 rank0 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0595 | Regression_eta: 0.0407 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0615 | LR: 4.96e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.3931s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank0 [2026-10-29 10:28:52,461] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 1% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:33:37,952] INFO : Step 27000/50000 rank0 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1017 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0512 | Regression_eta: 0.0322 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0545 | LR: 4.94e-04 | DataLoad Time: 0.0076s | Model Forward Time: 6.4463s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank0 [2026-10-29 10:33:38,195] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 53% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 52% 22% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 7% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 1% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +6 10 0x7408, 62036 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 24% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:33:38,228] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-27000.pth +rank0 [2026-10-29 10:33:38,817] INFO : Running validation on rank0 +rank0 [2026-10-29 10:33:38,817] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 10:33:38,817] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION | Step=27000/50000 | Train Loss=0.9700 | Valid Loss=0.9570 | Train Loss Unweighted=0.4718 | Valid Loss Unweighted=0.4697 | Stale=0 | ETA=1193.6m +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:34:34,338] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari=0.582459 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.699058 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_competing=0.792298 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.634787 +rank0 [2026-10-29 10:34:34,339] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.096732 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.103982 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/seconds_per_rank=22.0172 +rank0 [2026-10-29 10:34:34,340] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari=0.291664 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding_minus_raw/ari=0.161224 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 10:34:34,341] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 10:34:34,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:34:34,608] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 10:34:34,615] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 10:35:00,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 10:35:01,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 10:35:02,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 10:35:03,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 10:35:13,702] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:35:13,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:35:14,237] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 10:35:14,239] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:35:30,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 10:35:31,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 10:35:33,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 10:35:34,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 10:35:44,229] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:35:44,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:35:45,039] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 10:35:45,043] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 10:36:01,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 10:36:02,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 10:36:02,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 10:36:03,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 10:36:13,919] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 10:36:14,016] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits +rank0 [2026-10-29 10:36:17,317] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits +rank0 [2026-10-29 10:36:18,436] INFO : Plotted number of elements +rank0 [2026-10-29 10:36:20,008] INFO : Plotted jets +rank0 [2026-10-29 10:36:21,156] INFO : Plotted jet ratio +rank0 [2026-10-29 10:36:21,210] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:36:24,497] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 10:36:25,505] INFO : Plotted number of elements +rank0 [2026-10-29 10:36:27,027] INFO : Plotted jets +rank0 [2026-10-29 10:36:28,646] INFO : Plotted jet ratio +rank0 [2026-10-29 10:36:28,696] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits +rank0 [2026-10-29 10:36:31,058] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits +rank0 [2026-10-29 10:36:31,899] INFO : Plotted number of elements +rank0 [2026-10-29 10:36:33,406] INFO : Plotted jets +rank0 [2026-10-29 10:36:34,582] INFO : Plotted jet ratio +rank0 [2026-10-29 10:41:28,944] INFO : Step 27100/50000 rank0 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0675 | Regression_eta: 0.0425 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0537 | LR: 4.91e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.3325s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank0 [2026-10-29 10:41:29,190] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:46:23,210] INFO : Step 27200/50000 rank0 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0573 | Regression_eta: 0.0362 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0585 | LR: 4.88e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.7316s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank0 [2026-10-29 10:46:23,455] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 47.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 9% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 46% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:51:33,961] INFO : Step 27300/50000 rank0 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0586 | Regression_eta: 0.0384 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0590 | LR: 4.85e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3648s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank0 [2026-10-29 10:51:34,253] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:56:32,085] INFO : Step 27400/50000 rank0 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0592 | Regression_eta: 0.0472 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0695 | LR: 4.82e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6546s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank0 [2026-10-29 10:56:32,327] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:01:35,917] INFO : Step 27500/50000 rank0 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0604 | Regression_eta: 0.0493 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0661 | LR: 4.80e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.9782s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank0 [2026-10-29 11:01:36,159] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:08:29,761] INFO : Step 27600/50000 rank0 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0867 | Regression_eta: 0.0514 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0714 | LR: 4.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.0224s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank0 [2026-10-29 11:08:30,001] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 53.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:13:33,901] INFO : Step 27700/50000 rank0 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0242 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0512 | Regression_eta: 0.0438 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0459 | LR: 4.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6376s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank0 [2026-10-29 11:13:34,147] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:18:31,505] INFO : Step 27800/50000 rank0 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0522 | Regression_eta: 0.0499 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0609 | LR: 4.71e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.6892s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank0 [2026-10-29 11:18:31,750] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:23:39,167] INFO : Step 27900/50000 rank0 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0660 | Regression_eta: 0.0350 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0658 | LR: 4.68e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.6914s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank0 [2026-10-29 11:23:39,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +4 8 0x7408, 13395 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:28:55,590] INFO : Step 28000/50000 rank0 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1246 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0527 | Regression_eta: 0.0305 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0530 | LR: 4.66e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.6277s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank0 [2026-10-29 11:28:55,835] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 47.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +4 8 0x7408, 13395 53.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:28:55,961] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-28000.pth +rank0 [2026-10-29 11:28:56,259] INFO : Running validation on rank0 +rank0 [2026-10-29 11:28:56,259] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 11:28:56,259] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 11:29:48,491] INFO : VALIDATION | Step=28000/50000 | Train Loss=0.9048 | Valid Loss=0.9612 | Train Loss Unweighted=0.4299 | Valid Loss Unweighted=0.4616 | Stale=1 | ETA=1144.4m +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 11:29:48,492] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari=0.579686 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.710774 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_competing=0.807524 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.651841 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0939583 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.121037 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 11:29:48,493] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/seconds_per_rank=22.0819 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari=0.307298 +rank0 [2026-10-29 11:29:48,494] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 11:29:48,495] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding_minus_raw/ari=0.176858 +rank0 [2026-10-29 11:29:48,495] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 11:29:48,495] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 11:29:48,495] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 11:29:48,499] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:29:48,902] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 11:29:48,906] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 11:30:14,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 11:30:15,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 11:30:17,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 11:30:18,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 11:30:28,197] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 11:30:28,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:30:28,727] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 11:30:28,731] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:30:44,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 11:30:45,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 11:30:46,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 11:30:47,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 11:30:57,969] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 11:30:57,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:30:58,464] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 11:30:58,467] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 11:31:13,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 11:31:14,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 11:31:15,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 11:31:17,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 11:31:27,198] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 11:31:27,214] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits +rank0 [2026-10-29 11:31:30,507] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits +rank0 [2026-10-29 11:31:31,624] INFO : Plotted number of elements +rank0 [2026-10-29 11:31:33,187] INFO : Plotted jets +rank0 [2026-10-29 11:31:34,360] INFO : Plotted jet ratio +rank0 [2026-10-29 11:31:34,410] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:31:37,755] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 11:31:38,771] INFO : Plotted number of elements +rank0 [2026-10-29 11:31:40,283] INFO : Plotted jets +rank0 [2026-10-29 11:31:41,409] INFO : Plotted jet ratio +rank0 [2026-10-29 11:31:41,460] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits +rank0 [2026-10-29 11:31:43,969] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits +rank0 [2026-10-29 11:31:44,841] INFO : Plotted number of elements +rank0 [2026-10-29 11:31:46,888] INFO : Plotted jets +rank0 [2026-10-29 11:31:48,065] INFO : Plotted jet ratio +rank0 [2026-10-29 11:37:02,538] INFO : Step 28100/50000 rank0 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0559 | Regression_eta: 0.0299 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0614 | LR: 4.63e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.3157s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank0 [2026-10-29 11:37:02,785] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 53.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:42:06,920] INFO : Step 28200/50000 rank0 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0649 | Regression_eta: 0.0392 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0564 | LR: 4.60e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.9495s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank0 [2026-10-29 11:42:07,166] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 57.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 47.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:47:09,979] INFO : Step 28300/50000 rank0 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1121 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0674 | Regression_eta: 0.0446 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0691 | LR: 4.57e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.7941s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank0 [2026-10-29 11:47:10,224] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +2 6 0x7408, 15961 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 97% +4 8 0x7408, 13395 53.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 2% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 72% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:52:06,364] INFO : Step 28400/50000 rank0 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1633 | Classification_binary: 0.2713 | Regression_cos_phi: 0.0960 | Regression_energy: 0.1567 | Regression_eta: 0.0768 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0866 | LR: 4.55e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2856s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank0 [2026-10-29 11:52:06,611] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:56:57,595] INFO : Step 28500/50000 rank0 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0727 | Regression_eta: 0.0463 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0634 | LR: 4.52e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.7393s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank0 [2026-10-29 11:56:57,877] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 42% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:01:47,707] INFO : Step 28600/50000 rank0 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0504 | Regression_eta: 0.0309 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0542 | LR: 4.49e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.4433s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank0 [2026-10-29 12:01:47,957] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 98% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 1% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 44% 0% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 2% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 98% +6 10 0x7408, 62036 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:06:40,992] INFO : Step 28700/50000 rank0 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0650 | Regression_eta: 0.0437 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0689 | LR: 4.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.5121s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank0 [2026-10-29 12:06:41,236] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 54.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:11:31,778] INFO : Step 28800/50000 rank0 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0512 | Regression_eta: 0.0313 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0417 | LR: 4.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.6797s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank0 [2026-10-29 12:11:32,051] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 5% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 47.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 77% +4 8 0x7408, 13395 53.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:16:17,917] INFO : Step 28900/50000 rank0 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0501 | Regression_eta: 0.0696 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0417 | LR: 4.41e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.5205s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank0 [2026-10-29 12:16:18,159] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:21:10,969] INFO : Step 29000/50000 rank0 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0537 | Regression_eta: 0.0333 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0554 | LR: 4.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.8223s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank0 [2026-10-29 12:21:11,214] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 59.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +2 6 0x7408, 15961 47.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:21:11,238] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank0 [2026-10-29 12:21:11,701] INFO : Running validation on rank0 +rank0 [2026-10-29 12:21:11,701] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 12:21:11,701] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 12:22:04,757] INFO : VALIDATION | Step=29000/50000 | Train Loss=0.8982 | Valid Loss=0.9354 | Train Loss Unweighted=0.4359 | Valid Loss Unweighted=0.4537 | Stale=0 | ETA=1092.5m +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 12:22:04,758] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari=0.574634 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.684539 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_competing=0.772717 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.643524 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0889069 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.112719 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 12:22:04,759] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/seconds_per_rank=22.0651 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari=0.300596 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 12:22:04,760] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding_minus_raw/ari=0.170156 +rank0 [2026-10-29 12:22:04,761] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 12:22:04,761] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 12:22:04,761] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 12:22:04,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:22:05,023] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 12:22:05,026] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 12:22:31,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 12:22:32,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 12:22:33,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 12:22:35,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 12:22:45,438] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:22:46,055] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 12:22:46,058] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:23:02,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 12:23:03,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 12:23:04,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 12:23:05,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 12:23:15,778] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 12:23:15,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:23:16,443] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 12:23:16,447] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 12:23:32,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 12:23:33,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 12:23:34,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 12:23:35,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 12:23:45,171] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 12:23:45,186] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits +rank0 [2026-10-29 12:23:48,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits +rank0 [2026-10-29 12:23:49,506] INFO : Plotted number of elements +rank0 [2026-10-29 12:23:51,062] INFO : Plotted jets +rank0 [2026-10-29 12:23:52,217] INFO : Plotted jet ratio +rank0 [2026-10-29 12:23:52,269] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:23:55,496] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 12:23:56,530] INFO : Plotted number of elements +rank0 [2026-10-29 12:23:58,609] INFO : Plotted jets +rank0 [2026-10-29 12:23:59,784] INFO : Plotted jet ratio +rank0 [2026-10-29 12:23:59,848] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits +rank0 [2026-10-29 12:24:02,210] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits +rank0 [2026-10-29 12:24:03,047] INFO : Plotted number of elements +rank0 [2026-10-29 12:24:04,565] INFO : Plotted jets +rank0 [2026-10-29 12:24:05,756] INFO : Plotted jet ratio +rank0 [2026-10-29 12:28:58,475] INFO : Step 29100/50000 rank0 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0590 | Regression_eta: 0.0349 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0944 | LR: 4.35e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.8249s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank0 [2026-10-29 12:28:58,731] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 58.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:33:43,873] INFO : Step 29200/50000 rank0 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0517 | Regression_eta: 0.0725 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0698 | LR: 4.33e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.2462s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank0 [2026-10-29 12:47:12,901] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 112.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 40.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 44.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 44.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:49:26,805] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-29 14:49:26,853] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-29 14:49:54,071] INFO : Instantiating model +rank0 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-29 14:49:54,089] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank0 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank0 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank0 [2026-10-29 14:49:54,132] INFO : Initializing backbone layer 2 +rank0 [2026-10-29 14:49:54,140] INFO : Initializing backbone layer 3 +rank0 [2026-10-29 14:49:54,149] INFO : Initializing backbone layer 4 +rank0 [2026-10-29 14:49:54,157] INFO : Initializing backbone layer 5 +rank0 [2026-10-29 14:49:54,165] INFO : Backbone initialization took 0.08s +rank0 [2026-10-29 14:49:54,166] INFO : backbone parameters: 7901376 +rank0 [2026-10-29 14:49:54,166] INFO : Initializing output DNNs +rank0 [2026-10-29 14:49:54,202] INFO : Output DNNs initialization took 0.04s +rank0 [2026-10-29 14:49:54,202] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-29 14:49:54,202] INFO : jagged_attention enabled=False +rank0 [2026-10-29 14:49:54,202] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-29 14:49:54,202] INFO : task_queries enabled=False +rank0 [2026-10-29 14:49:54,202] INFO : classification_norm parameters: 0 +rank0 [2026-10-29 14:49:54,202] INFO : regression_norm parameters: 0 +rank0 [2026-10-29 14:49:54,202] INFO : classification_query parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : regression_query parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : classification_readout parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : regression_readout parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : output_mode=set +rank0 [2026-10-29 14:49:54,203] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-29 14:49:54,203] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_pid parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_pt parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_eta parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-29 14:49:54,203] INFO : nn_energy parameters: 0 +rank0 [2026-10-29 14:49:54,204] INFO : Total MLPF parameters: 12786964 +rank0 [2026-10-29 14:49:54,204] INFO : MLPF __init__ done +rank0 [2026-10-29 14:49:54,204] INFO : Instantiated model +rank0 [2026-10-29 14:49:54,204] INFO : Moving model to device rank=0 +rank0 [2026-10-29 14:49:54,229] INFO : Moved model to device rank=0 +rank0 [2026-10-29 14:49:54,231] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-29 14:49:54,793] INFO : Restoring training from step 29001 +rank0 [2026-10-29 14:49:54,801] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank0 [2026-10-29 14:49:54,802] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-29 14:49:55,512] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank0 [2026-10-29 14:49:55,512] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-29 14:49:55,514] INFO : Trainable parameters: 12786964 +rank0 [2026-10-29 14:49:55,514] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-29 14:49:55,514] INFO : Total parameters: 12810232 +rank0 [2026-10-29 14:49:55,518] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-29 14:49:55,523] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645 +rank0 [2026-10-29 14:49:55,523] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645 +rank0 [2026-10-29 14:49:55,523] INFO : Getting dataloaders +rank0 [2026-10-29 14:49:55,523] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:00,257] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,355] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,461] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,561] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,655] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:00,748] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,121] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,200] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,279] INFO : train_dataset: clic_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:50:01,363] INFO : train_dataset: clic_edm_ttbar_hits, 90900 +rank0 [2026-10-29 14:50:01,363] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:01,458] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,565] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,649] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,748] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,843] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:01,940] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,037] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,139] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,228] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:50:02,347] INFO : train_dataset: clic_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-29 14:50:02,347] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:02,440] INFO : train_dataset: clic_edm_qq_hits, 90000 +rank0 [2026-10-29 14:50:02,861] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-29 14:50:02,939] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-29 14:50:03,017] INFO : train_dataset: clic_edm_qq_hits, 89999 +rank0 [2026-10-29 14:50:03,092] INFO : train_dataset: clic_edm_qq_hits, 89997 +rank0 [2026-10-29 14:50:03,173] INFO : train_dataset: clic_edm_qq_hits, 89996 +rank0 [2026-10-29 14:50:03,255] INFO : train_dataset: clic_edm_qq_hits, 89998 +rank0 [2026-10-29 14:50:03,364] INFO : train_dataset: clic_edm_qq_hits, 89994 +rank0 [2026-10-29 14:50:03,471] INFO : train_dataset: clic_edm_qq_hits, 89995 +rank0 [2026-10-29 14:50:03,550] INFO : train_dataset: clic_edm_qq_hits, 90099 +rank0 [2026-10-29 14:50:03,550] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-29 14:50:03,551] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-29 14:50:03,551] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-29 14:50:03,551] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:03,577] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,603] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,629] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,656] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,683] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,710] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,736] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,762] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,789] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,815] INFO : valid_dataset: clic_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:50:03,815] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:03,841] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,868] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,897] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,926] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,953] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:03,984] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,016] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,041] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,066] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,091] INFO : valid_dataset: clic_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:50:04,091] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:50:04,115] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,140] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,164] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,196] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,220] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,245] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,270] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,294] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,319] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,345] INFO : valid_dataset: clic_edm_qq_hits, 51 +rank0 [2026-10-29 14:50:04,345] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-29 14:50:04,345] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-29 14:50:18,099] INFO : Got dataloaders +rank0 [2026-10-29 14:50:18,099] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank0 [2026-10-29 14:50:18,099] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank0 [2026-10-29 14:50:18,100] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank0 [2026-10-29 14:50:18,100] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank0 [2026-10-29 14:50:20,086] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-29 14:50:20,086] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-29 14:50:20,086] INFO : Starting training from step 29001 to 50000 on rank 0 of 8 +rank0 [2026-10-29 14:50:20,092] INFO : Creating train_iterator +rank0 [2026-10-29 14:50:20,092] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank0 [2026-10-29 14:50:34,100] INFO : Created train_iterator +rank0 [2026-10-29 14:55:13,601] INFO : Step 29100/50000 rank0 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0487 | Regression_eta: 0.0390 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0782 | LR: 4.35e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3964s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank0 [2026-10-29 14:55:13,838] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 193.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 198.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 50.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:59:47,486] INFO : Step 29200/50000 rank0 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0492 | Regression_eta: 0.0632 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0588 | LR: 4.33e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0606s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank0 [2026-10-29 14:59:47,724] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:04:22,483] INFO : Step 29300/50000 rank0 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0254 | Classification_binary: 0.1091 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0622 | Regression_eta: 0.0275 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0309 | LR: 4.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7398s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank0 [2026-10-29 15:04:22,729] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 50.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:08:57,678] INFO : Step 29400/50000 rank0 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0603 | Regression_eta: 0.0416 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0690 | LR: 4.27e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9183s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank0 [2026-10-29 15:08:57,914] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:13:33,671] INFO : Step 29500/50000 rank0 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0509 | Regression_eta: 0.0344 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0658 | LR: 4.24e-04 | DataLoad Time: 0.0152s | Model Forward Time: 7.1586s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank0 [2026-10-29 15:13:33,930] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:18:12,264] INFO : Step 29600/50000 rank0 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1299 | Regression_cos_phi: 0.1125 | Regression_energy: 0.0603 | Regression_eta: 0.0437 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0578 | LR: 4.22e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3792s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank0 [2026-10-29 15:18:12,509] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:22:46,267] INFO : Step 29700/50000 rank0 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0549 | Regression_eta: 0.0387 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0561 | LR: 4.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.3126s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank0 [2026-10-29 15:22:46,507] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:27:19,223] INFO : Step 29800/50000 rank0 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0580 | Regression_eta: 0.0480 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0788 | LR: 4.16e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2912s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank0 [2026-10-29 15:27:19,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:31:52,607] INFO : Step 29900/50000 rank0 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0737 | Regression_eta: 0.0415 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0685 | LR: 4.14e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2795s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank0 [2026-10-29 15:31:52,847] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:36:27,498] INFO : Step 30000/50000 rank0 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0700 | Regression_eta: 0.0344 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0565 | LR: 4.11e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1613s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank0 [2026-10-29 15:36:27,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:36:27,760] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-30000.pth +rank0 [2026-10-29 15:36:28,089] INFO : Running validation on rank0 +rank0 [2026-10-29 15:36:28,089] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 15:36:28,089] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 15:37:19,837] INFO : VALIDATION | Step=30000/50000 | Train Loss=0.8609 | Valid Loss=0.9317 | Train Loss Unweighted=0.4144 | Valid Loss Unweighted=0.4504 | Stale=0 | ETA=31.3m +rank0 [2026-10-29 15:37:19,837] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:37:19,837] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari=0.587288 +rank0 [2026-10-29 15:37:19,838] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.704007 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_competing=0.79782 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.654991 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.101561 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.124186 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 15:37:19,839] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/seconds_per_rank=21.9521 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari=0.296322 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding_minus_raw/ari=0.165882 +rank0 [2026-10-29 15:37:19,840] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 15:37:19,841] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 15:37:19,841] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 15:37:19,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:37:20,099] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 15:37:20,124] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 15:37:50,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 15:37:52,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 15:37:53,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 15:37:54,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 15:38:04,363] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:38:04,946] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 15:38:04,950] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:38:21,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 15:38:22,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 15:38:23,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 15:38:24,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 15:38:34,516] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:38:35,046] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 15:38:35,050] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 15:38:50,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 15:38:51,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 15:38:52,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 15:38:53,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 15:39:03,693] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 15:39:03,709] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits +rank0 [2026-10-29 15:39:06,955] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits +rank0 [2026-10-29 15:39:08,291] INFO : Plotted number of elements +rank0 [2026-10-29 15:39:10,199] INFO : Plotted jets +rank0 [2026-10-29 15:39:11,345] INFO : Plotted jet ratio +rank0 [2026-10-29 15:39:11,414] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:39:14,676] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 15:39:15,654] INFO : Plotted number of elements +rank0 [2026-10-29 15:39:17,135] INFO : Plotted jets +rank0 [2026-10-29 15:39:18,258] INFO : Plotted jet ratio +rank0 [2026-10-29 15:39:18,312] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits +rank0 [2026-10-29 15:39:20,757] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits +rank0 [2026-10-29 15:39:21,592] INFO : Plotted number of elements +rank0 [2026-10-29 15:39:23,050] INFO : Plotted jets +rank0 [2026-10-29 15:39:24,688] INFO : Plotted jet ratio +rank0 [2026-10-29 15:44:19,859] INFO : Step 30100/50000 rank0 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0639 | Regression_eta: 0.0404 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0603 | LR: 4.08e-04 | DataLoad Time: 0.0078s | Model Forward Time: 5.4043s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank0 [2026-10-29 15:44:20,101] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 47.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:49:02,212] INFO : Step 30200/50000 rank0 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0641 | Regression_eta: 0.0327 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0445 | LR: 4.06e-04 | DataLoad Time: 0.0281s | Model Forward Time: 5.6006s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank0 [2026-10-29 15:49:02,451] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 83% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 34% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:53:39,523] INFO : Step 30300/50000 rank0 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0472 | Regression_eta: 0.0404 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0618 | LR: 4.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9912s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank0 [2026-10-29 15:53:39,760] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:58:14,108] INFO : Step 30400/50000 rank0 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0543 | Regression_eta: 0.0436 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0699 | LR: 4.00e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1709s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank0 [2026-10-29 15:58:14,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:02:52,164] INFO : Step 30500/50000 rank0 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.2506 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0829 | Regression_eta: 0.0537 | Regression_pt: 0.0860 | Regression_sin_phi: 0.1045 | LR: 3.98e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.1330s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank0 [2026-10-29 16:02:52,415] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:07:32,698] INFO : Step 30600/50000 rank0 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1266 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0747 | Regression_eta: 0.0502 | Regression_pt: 0.0782 | Regression_sin_phi: 0.1099 | LR: 3.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5863s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank0 [2026-10-29 16:07:32,935] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:12:19,458] INFO : Step 30700/50000 rank0 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0666 | Regression_eta: 0.0409 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0849 | LR: 3.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9593s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank0 [2026-10-29 16:12:19,703] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:17:04,279] INFO : Step 30800/50000 rank0 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0623 | Regression_eta: 0.0690 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0826 | LR: 3.90e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.7071s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank0 [2026-10-29 16:17:04,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:21:39,220] INFO : Step 30900/50000 rank0 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0556 | Regression_eta: 0.0353 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0543 | LR: 3.87e-04 | DataLoad Time: 0.0080s | Model Forward Time: 5.4564s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank0 [2026-10-29 16:21:39,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:26:03,425] INFO : Step 31000/50000 rank0 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1097 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0497 | Regression_eta: 0.0293 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0290 | LR: 3.84e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6808s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank0 [2026-10-29 16:26:49,211] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 44.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 44.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 42.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 43.0°C 114.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:26:49,234] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-31000.pth +rank0 [2026-10-29 16:26:49,555] INFO : Running validation on rank0 +rank0 [2026-10-29 16:26:49,555] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:26:49,556] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION | Step=31000/50000 | Train Loss=0.8308 | Valid Loss=0.9055 | Train Loss Unweighted=0.4031 | Valid Loss Unweighted=0.4287 | Stale=0 | ETA=59.7m +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:27:40,342] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari=0.590406 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.709999 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_competing=0.804229 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.650153 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.104678 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.119349 +rank0 [2026-10-29 16:27:40,343] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/seconds_per_rank=21.6806 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:27:40,344] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari=0.301608 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding_minus_raw/ari=0.171169 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 16:27:40,345] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:27:40,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:27:40,685] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:27:40,704] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 16:28:08,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:28:09,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:28:11,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:28:13,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:28:23,738] INFO : Time taken to make predictions on device 0 is: 0.30 min +rank0 [2026-10-29 16:28:24,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:28:24,725] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:28:24,729] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:28:40,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:28:41,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:28:43,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:28:44,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:28:54,177] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:28:54,191] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:28:54,676] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 16:28:54,680] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 16:29:10,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:29:11,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:29:12,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:29:13,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:29:23,192] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 16:29:23,208] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:29:26,549] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits +rank0 [2026-10-29 16:29:27,617] INFO : Plotted number of elements +rank0 [2026-10-29 16:29:29,183] INFO : Plotted jets +rank0 [2026-10-29 16:29:30,343] INFO : Plotted jet ratio +rank0 [2026-10-29 16:29:30,402] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:29:33,819] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 16:29:34,854] INFO : Plotted number of elements +rank0 [2026-10-29 16:29:36,859] INFO : Plotted jets +rank0 [2026-10-29 16:29:38,065] INFO : Plotted jet ratio +rank0 [2026-10-29 16:29:38,137] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits +rank0 [2026-10-29 16:29:40,574] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits +rank0 [2026-10-29 16:29:41,436] INFO : Plotted number of elements +rank0 [2026-10-29 16:29:43,004] INFO : Plotted jets +rank0 [2026-10-29 16:29:44,184] INFO : Plotted jet ratio +rank0 [2026-10-29 16:34:38,234] INFO : Step 31100/50000 rank0 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0558 | Regression_eta: 0.0328 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0526 | LR: 3.82e-04 | DataLoad Time: 0.0208s | Model Forward Time: 6.2276s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank0 [2026-10-29 16:34:38,475] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +4 8 0x7408, 13395 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:39:14,572] INFO : Step 31200/50000 rank0 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0516 | Regression_eta: 0.0449 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0509 | LR: 3.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.0181s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank0 [2026-10-29 16:39:14,811] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:43:57,874] INFO : Step 31300/50000 rank0 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1169 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0614 | Regression_eta: 0.0334 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0471 | LR: 3.76e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5445s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank0 [2026-10-29 16:43:58,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:48:31,342] INFO : Step 31400/50000 rank0 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0571 | Regression_eta: 0.0515 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0566 | LR: 3.74e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.4574s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank0 [2026-10-29 16:48:31,587] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:53:08,452] INFO : Step 31500/50000 rank0 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0536 | Regression_eta: 0.0557 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0562 | LR: 3.71e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1913s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank0 [2026-10-29 16:53:08,694] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:57:39,265] INFO : Step 31600/50000 rank0 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0348 | Regression_energy: 0.0503 | Regression_eta: 0.0269 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0379 | LR: 3.69e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8240s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank0 [2026-10-29 16:57:39,503] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:02:12,163] INFO : Step 31700/50000 rank0 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0709 | Regression_eta: 0.0380 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0782 | LR: 3.66e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7874s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank0 [2026-10-29 17:02:12,399] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:06:48,215] INFO : Step 31800/50000 rank0 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0512 | Regression_eta: 0.0413 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0624 | LR: 3.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9246s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank0 [2026-10-29 17:06:48,453] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:11:28,054] INFO : Step 31900/50000 rank0 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0566 | Regression_eta: 0.0354 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0576 | LR: 3.61e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1957s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank0 [2026-10-29 17:11:28,304] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:16:12,678] INFO : Step 32000/50000 rank0 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0484 | Regression_eta: 0.0453 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0428 | LR: 3.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1379s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank0 [2026-10-29 17:16:12,933] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:16:12,956] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-32000.pth +rank0 [2026-10-29 17:16:13,378] INFO : Running validation on rank0 +rank0 [2026-10-29 17:16:13,378] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 17:16:13,378] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 17:17:04,387] INFO : VALIDATION | Step=32000/50000 | Train Loss=0.8649 | Valid Loss=0.9043 | Train Loss Unweighted=0.4265 | Valid Loss Unweighted=0.4335 | Stale=0 | ETA=82.5m +rank0 [2026-10-29 17:17:04,387] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:17:04,387] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 17:17:04,387] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari=0.576942 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.717676 +rank0 [2026-10-29 17:17:04,388] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_competing=0.814292 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.657791 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.091215 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.126986 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 17:17:04,389] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/seconds_per_rank=21.8222 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari=0.313769 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding_minus_raw/ari=0.183329 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 17:17:04,390] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 17:17:04,396] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:17:04,718] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 17:17:04,723] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 17:17:32,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 17:17:33,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 17:17:34,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 17:17:35,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 17:17:45,776] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:17:46,350] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 17:17:46,352] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:18:02,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 17:18:03,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 17:18:04,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 17:18:05,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 17:18:15,909] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:18:15,920] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:18:16,464] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 17:18:16,468] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 17:18:31,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 17:18:32,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 17:18:33,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 17:18:34,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 17:18:44,988] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 17:18:45,004] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits +rank0 [2026-10-29 17:18:48,317] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits +rank0 [2026-10-29 17:18:49,389] INFO : Plotted number of elements +rank0 [2026-10-29 17:18:50,947] INFO : Plotted jets +rank0 [2026-10-29 17:18:52,464] INFO : Plotted jet ratio +rank0 [2026-10-29 17:18:52,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:18:55,825] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 17:18:56,811] INFO : Plotted number of elements +rank0 [2026-10-29 17:18:58,317] INFO : Plotted jets +rank0 [2026-10-29 17:18:59,451] INFO : Plotted jet ratio +rank0 [2026-10-29 17:18:59,501] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits +rank0 [2026-10-29 17:19:01,985] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits +rank0 [2026-10-29 17:19:02,844] INFO : Plotted number of elements +rank0 [2026-10-29 17:19:04,312] INFO : Plotted jets +rank0 [2026-10-29 17:19:05,486] INFO : Plotted jet ratio +rank0 [2026-10-29 17:23:52,061] INFO : Step 32100/50000 rank0 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0485 | Regression_eta: 0.0288 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0641 | LR: 3.56e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6302s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank0 [2026-10-29 17:23:52,298] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:28:36,853] INFO : Step 32200/50000 rank0 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0446 | Regression_eta: 0.0267 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0491 | LR: 3.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.1234s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank0 [2026-10-29 17:28:37,115] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:33:22,217] INFO : Step 32300/50000 rank0 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0511 | Regression_eta: 0.0286 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0664 | LR: 3.51e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.7120s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank0 [2026-10-29 17:33:22,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +4 8 0x7408, 13395 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:38:09,075] INFO : Step 32400/50000 rank0 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0524 | Regression_eta: 0.0319 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0421 | LR: 3.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.5371s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank0 [2026-10-29 17:38:09,323] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 98% +4 8 0x7408, 13395 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 69% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:42:53,462] INFO : Step 32500/50000 rank0 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0479 | Regression_eta: 0.0337 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0633 | LR: 3.46e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.5165s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank0 [2026-10-29 17:42:53,709] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:47:37,310] INFO : Step 32600/50000 rank0 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0281 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0593 | Regression_eta: 0.0375 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0565 | LR: 3.43e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.3321s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank0 [2026-10-29 17:47:37,563] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:52:17,925] INFO : Step 32700/50000 rank0 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0476 | Regression_eta: 0.0392 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0581 | LR: 3.41e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3230s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank0 [2026-10-29 17:52:18,168] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:57:02,545] INFO : Step 32800/50000 rank0 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0581 | Regression_eta: 0.0471 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0557 | LR: 3.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2642s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank0 [2026-10-29 17:57:02,789] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:01:38,901] INFO : Step 32900/50000 rank0 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0510 | Regression_eta: 0.0330 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0686 | LR: 3.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1783s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank0 [2026-10-29 18:01:39,141] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:06:23,262] INFO : Step 33000/50000 rank0 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0250 | Classification_binary: 0.0979 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0511 | Regression_eta: 0.0246 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0604 | LR: 3.33e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8609s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank0 [2026-10-29 18:06:23,514] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:06:23,537] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-33000.pth +rank0 [2026-10-29 18:06:23,952] INFO : Running validation on rank0 +rank0 [2026-10-29 18:06:23,952] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:06:23,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION | Step=33000/50000 | Train Loss=0.8328 | Valid Loss=0.8968 | Train Loss Unweighted=0.3978 | Valid Loss Unweighted=0.4310 | Stale=0 | ETA=101.4m +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 18:07:15,756] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari=0.588857 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.707586 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_competing=0.80279 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.654841 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.10313 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.124036 +rank0 [2026-10-29 18:07:15,757] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/seconds_per_rank=21.813 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 18:07:15,758] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari=0.317811 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding_minus_raw/ari=0.187371 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 18:07:15,759] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:07:15,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:07:16,021] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:07:16,030] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 18:07:44,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:07:45,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:07:46,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:07:47,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:07:57,880] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:07:57,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:07:58,401] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:07:58,404] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:08:14,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:08:15,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:08:16,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:08:17,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:08:27,833] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:08:27,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:08:28,344] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 18:08:28,347] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 18:08:43,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:08:44,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:08:45,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:08:46,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:08:56,853] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 18:08:56,869] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:09:00,087] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:09:01,173] INFO : Plotted number of elements +rank0 [2026-10-29 18:09:02,795] INFO : Plotted jets +rank0 [2026-10-29 18:09:04,026] INFO : Plotted jet ratio +rank0 [2026-10-29 18:09:04,162] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:09:07,471] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:09:08,459] INFO : Plotted number of elements +rank0 [2026-10-29 18:09:09,962] INFO : Plotted jets +rank0 [2026-10-29 18:09:11,087] INFO : Plotted jet ratio +rank0 [2026-10-29 18:09:11,151] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits +rank0 [2026-10-29 18:09:13,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits +rank0 [2026-10-29 18:09:14,727] INFO : Plotted number of elements +rank0 [2026-10-29 18:09:16,237] INFO : Plotted jets +rank0 [2026-10-29 18:09:17,434] INFO : Plotted jet ratio +rank0 [2026-10-29 18:13:51,990] INFO : Step 33100/50000 rank0 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0511 | Regression_eta: 0.0465 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0614 | LR: 3.31e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9176s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank0 [2026-10-29 18:13:52,239] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:18:34,620] INFO : Step 33200/50000 rank0 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0499 | Regression_eta: 0.0517 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0844 | LR: 3.28e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0730s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank0 [2026-10-29 18:18:34,855] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:23:16,094] INFO : Step 33300/50000 rank0 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0565 | Regression_eta: 0.0373 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0609 | LR: 3.26e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.2458s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank0 [2026-10-29 18:23:16,343] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:27:59,942] INFO : Step 33400/50000 rank0 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0487 | Regression_eta: 0.0295 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0544 | LR: 3.23e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.9147s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank0 [2026-10-29 18:28:00,183] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:32:39,037] INFO : Step 33500/50000 rank0 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0476 | Regression_eta: 0.0441 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0666 | LR: 3.21e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0858s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank0 [2026-10-29 18:32:39,277] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:37:28,430] INFO : Step 33600/50000 rank0 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0653 | Regression_eta: 0.0394 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0608 | LR: 3.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0230s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank0 [2026-10-29 18:37:28,668] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:42:20,881] INFO : Step 33700/50000 rank0 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.0986 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0400 | Regression_eta: 0.0233 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0574 | LR: 3.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1538s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank0 [2026-10-29 18:42:21,129] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:47:05,640] INFO : Step 33800/50000 rank0 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0599 | Regression_eta: 0.0489 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0777 | LR: 3.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0480s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank0 [2026-10-29 18:47:05,881] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:51:53,006] INFO : Step 33900/50000 rank0 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0367 | Regression_eta: 0.0397 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0535 | LR: 3.11e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9514s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank0 [2026-10-29 18:51:53,244] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:56:36,487] INFO : Step 34000/50000 rank0 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0424 | Regression_eta: 0.0486 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0578 | LR: 3.09e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8421s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank0 [2026-10-29 18:56:36,726] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:56:36,828] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-34000.pth +rank0 [2026-10-29 18:56:37,118] INFO : Running validation on rank0 +rank0 [2026-10-29 18:56:37,118] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:56:37,118] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:57:30,379] INFO : VALIDATION | Step=34000/50000 | Train Loss=0.8429 | Valid Loss=0.8880 | Train Loss Unweighted=0.3992 | Valid Loss Unweighted=0.4147 | Stale=0 | ETA=116.3m +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 18:57:30,380] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari=0.5945 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.728176 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_competing=0.82693 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.66193 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.108773 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.131126 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 18:57:30,381] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/seconds_per_rank=21.6753 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari=0.29859 +rank0 [2026-10-29 18:57:30,382] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 18:57:30,383] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding_minus_raw/ari=0.16815 +rank0 [2026-10-29 18:57:30,383] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 18:57:30,383] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 18:57:30,383] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:57:30,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:57:30,641] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:57:30,655] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 18:57:55,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:57:56,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:57:57,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:57:58,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:58:08,716] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:58:08,732] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:58:09,211] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:58:09,214] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:58:25,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:58:26,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:58:28,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:58:29,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:58:39,249] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:58:39,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:58:39,751] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 18:58:39,755] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 18:58:55,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:58:56,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:58:57,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:58:58,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:59:08,165] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 18:59:08,180] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:59:11,495] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits +rank0 [2026-10-29 18:59:12,944] INFO : Plotted number of elements +rank0 [2026-10-29 18:59:14,450] INFO : Plotted jets +rank0 [2026-10-29 18:59:15,589] INFO : Plotted jet ratio +rank0 [2026-10-29 18:59:15,657] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:59:18,875] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 18:59:19,851] INFO : Plotted number of elements +rank0 [2026-10-29 18:59:21,363] INFO : Plotted jets +rank0 [2026-10-29 18:59:22,485] INFO : Plotted jet ratio +rank0 [2026-10-29 18:59:22,546] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits +rank0 [2026-10-29 18:59:24,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits +rank0 [2026-10-29 18:59:25,839] INFO : Plotted number of elements +rank0 [2026-10-29 18:59:27,343] INFO : Plotted jets +rank0 [2026-10-29 18:59:28,997] INFO : Plotted jet ratio +rank0 [2026-10-29 19:04:17,137] INFO : Step 34100/50000 rank0 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0524 | Regression_eta: 0.0426 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0712 | LR: 3.06e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.4885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank0 [2026-10-29 19:04:17,375] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:09:15,318] INFO : Step 34200/50000 rank0 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0603 | Regression_eta: 0.0402 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0700 | LR: 3.04e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.3764s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank0 [2026-10-29 19:09:15,570] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:14:02,368] INFO : Step 34300/50000 rank0 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0503 | Regression_eta: 0.0443 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0476 | LR: 3.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9553s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank0 [2026-10-29 19:14:02,611] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:18:48,882] INFO : Step 34400/50000 rank0 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0454 | Regression_eta: 0.0352 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0590 | LR: 2.99e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6027s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank0 [2026-10-29 19:19:06,162] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 47.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 45.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 46.0°C 116.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:23:50,140] INFO : Step 34500/50000 rank0 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0535 | Regression_eta: 0.0405 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0717 | LR: 2.97e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.9812s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank0 [2026-10-29 19:23:50,381] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 5% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 30% +2 6 0x7408, 15961 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 94% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 0% +4 8 0x7408, 13395 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 1% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 88% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 81% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:28:35,423] INFO : Step 34600/50000 rank0 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0480 | Regression_eta: 0.0412 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0615 | LR: 2.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1204s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank0 [2026-10-29 19:28:35,667] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:33:36,497] INFO : Step 34700/50000 rank0 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.0906 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0579 | Regression_eta: 0.0309 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0324 | LR: 2.92e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9018s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank0 [2026-10-29 19:33:36,736] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:38:27,265] INFO : Step 34800/50000 rank0 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0238 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0473 | Regression_eta: 0.0205 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0348 | LR: 2.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7137s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank0 [2026-10-29 19:38:27,516] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:43:16,263] INFO : Step 34900/50000 rank0 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0436 | Regression_eta: 0.0389 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0517 | LR: 2.88e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9651s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank0 [2026-10-29 19:43:16,501] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:48:20,988] INFO : Step 35000/50000 rank0 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0553 | Regression_eta: 0.0365 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0388 | LR: 2.85e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0252s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank0 [2026-10-29 19:48:21,227] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:48:21,355] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 19:48:21,641] INFO : Running validation on rank0 +rank0 [2026-10-29 19:48:21,641] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 19:48:21,641] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 19:49:15,982] INFO : VALIDATION | Step=35000/50000 | Train Loss=0.8846 | Valid Loss=0.8640 | Train Loss Unweighted=0.4004 | Valid Loss Unweighted=0.4074 | Stale=0 | ETA=128.1m +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 19:49:15,983] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari=0.592919 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.723995 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_competing=0.824044 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.655229 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.107191 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.124424 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 19:49:15,984] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/seconds_per_rank=21.6715 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari=0.302102 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding_minus_raw/ari=0.171662 +rank0 [2026-10-29 19:49:15,985] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 19:49:15,986] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 19:49:15,986] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 19:49:15,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:49:16,242] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 19:49:16,246] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 19:49:42,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 19:49:43,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 19:49:45,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 19:49:46,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 19:49:56,249] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 19:49:56,263] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:49:56,734] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 19:49:56,738] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:50:12,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 19:50:14,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 19:50:15,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 19:50:16,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 19:50:26,276] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:50:26,882] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 19:50:26,886] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 19:50:42,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 19:50:43,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 19:50:44,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 19:50:45,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 19:50:55,353] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 19:50:55,496] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits +rank0 [2026-10-29 19:50:58,871] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits +rank0 [2026-10-29 19:50:59,947] INFO : Plotted number of elements +rank0 [2026-10-29 19:51:01,502] INFO : Plotted jets +rank0 [2026-10-29 19:51:02,667] INFO : Plotted jet ratio +rank0 [2026-10-29 19:51:02,746] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:51:05,876] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 19:51:06,858] INFO : Plotted number of elements +rank0 [2026-10-29 19:51:08,362] INFO : Plotted jets +rank0 [2026-10-29 19:51:09,921] INFO : Plotted jet ratio +rank0 [2026-10-29 19:51:09,971] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits +rank0 [2026-10-29 19:51:12,272] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits +rank0 [2026-10-29 19:51:13,137] INFO : Plotted number of elements +rank0 [2026-10-29 19:51:14,647] INFO : Plotted jets +rank0 [2026-10-29 19:51:15,859] INFO : Plotted jet ratio +rank0 [2026-10-29 19:56:08,330] INFO : Step 35100/50000 rank0 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0489 | Regression_eta: 0.0349 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0470 | LR: 2.83e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1233s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank0 [2026-10-29 19:56:08,568] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:01:02,310] INFO : Step 35200/50000 rank0 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0548 | Regression_eta: 0.0237 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0293 | LR: 2.81e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3611s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank0 [2026-10-29 20:01:02,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 90% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:05:52,900] INFO : Step 35300/50000 rank0 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0613 | Regression_eta: 0.0284 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0676 | LR: 2.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8900s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank0 [2026-10-29 20:05:53,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +2 6 0x7408, 15961 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 8% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 98% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 65% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 18% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:10:55,043] INFO : Step 35400/50000 rank0 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1215 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0637 | Regression_eta: 0.0310 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0387 | LR: 2.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0676s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank0 [2026-10-29 20:10:55,285] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:15:43,082] INFO : Step 35500/50000 rank0 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0400 | Regression_eta: 0.0254 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0725 | LR: 2.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1496s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank0 [2026-10-29 20:15:43,319] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:20:38,652] INFO : Step 35600/50000 rank0 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0551 | Regression_eta: 0.0301 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0562 | LR: 2.72e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7968s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank0 [2026-10-29 20:20:38,892] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:25:33,876] INFO : Step 35700/50000 rank0 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0591 | Regression_eta: 0.0491 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0861 | LR: 2.70e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.9138s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank0 [2026-10-29 20:25:34,116] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:30:22,737] INFO : Step 35800/50000 rank0 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0396 | Regression_eta: 0.0348 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0505 | LR: 2.68e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9254s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank0 [2026-10-29 20:30:22,974] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:35:21,078] INFO : Step 35900/50000 rank0 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0652 | Regression_eta: 0.0416 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0599 | LR: 2.65e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.4901s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank0 [2026-10-29 20:35:21,324] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 91% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 3% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 62% +4 8 0x7408, 13395 47.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 96% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 75% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 36% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:40:16,749] INFO : Step 36000/50000 rank0 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0486 | Regression_eta: 0.0373 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0594 | LR: 2.63e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9307s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank0 [2026-10-29 20:40:16,988] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 48.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:40:17,081] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-36000.pth +rank0 [2026-10-29 20:40:17,378] INFO : Running validation on rank0 +rank0 [2026-10-29 20:40:17,378] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 20:40:17,378] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION | Step=36000/50000 | Train Loss=0.8271 | Valid Loss=0.8372 | Train Loss Unweighted=0.3970 | Valid Loss Unweighted=0.3980 | Stale=0 | ETA=136.4m +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 20:41:08,957] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari=0.595194 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.734208 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_competing=0.835289 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.664581 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.109467 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.133777 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 20:41:08,958] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/seconds_per_rank=21.5566 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 20:41:08,959] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari=0.299015 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding_minus_raw/ari=0.168576 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 20:41:08,960] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 20:41:08,967] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:41:09,219] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 20:41:09,233] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 20:41:36,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 20:41:37,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 20:41:38,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 20:41:39,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 20:41:49,772] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:41:49,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:41:50,341] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 20:41:50,345] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:42:06,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 20:42:07,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 20:42:09,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 20:42:10,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 20:42:20,172] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:42:20,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:42:20,694] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 20:42:20,698] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 20:42:36,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 20:42:37,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 20:42:38,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 20:42:39,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 20:42:49,298] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 20:42:49,314] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits +rank0 [2026-10-29 20:42:52,544] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits +rank0 [2026-10-29 20:42:53,608] INFO : Plotted number of elements +rank0 [2026-10-29 20:42:55,162] INFO : Plotted jets +rank0 [2026-10-29 20:42:56,331] INFO : Plotted jet ratio +rank0 [2026-10-29 20:42:56,407] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:42:59,594] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 20:43:00,599] INFO : Plotted number of elements +rank0 [2026-10-29 20:43:02,102] INFO : Plotted jets +rank0 [2026-10-29 20:43:03,671] INFO : Plotted jet ratio +rank0 [2026-10-29 20:43:03,722] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits +rank0 [2026-10-29 20:43:05,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits +rank0 [2026-10-29 20:43:06,724] INFO : Plotted number of elements +rank0 [2026-10-29 20:43:08,228] INFO : Plotted jets +rank0 [2026-10-29 20:43:09,416] INFO : Plotted jet ratio +rank0 [2026-10-29 20:47:47,670] INFO : Step 36100/50000 rank0 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1098 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0391 | Regression_eta: 0.0369 | Regression_pt: 0.0390 | Regression_sin_phi: 0.0545 | LR: 2.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1041s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank0 [2026-10-29 20:47:47,922] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:52:23,530] INFO : Step 36200/50000 rank0 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0432 | Regression_eta: 0.0323 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0500 | LR: 2.59e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.0852s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank0 [2026-10-29 20:52:23,768] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:57:00,344] INFO : Step 36300/50000 rank0 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0486 | Regression_eta: 0.0381 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0542 | LR: 2.57e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.9879s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank0 [2026-10-29 20:57:00,582] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:01:39,016] INFO : Step 36400/50000 rank0 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0689 | Regression_eta: 0.0283 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0466 | LR: 2.55e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5396s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank0 [2026-10-29 21:01:39,265] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:06:15,016] INFO : Step 36500/50000 rank0 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0497 | Regression_eta: 0.0284 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0631 | LR: 2.52e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1925s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank0 [2026-10-29 21:06:15,255] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:10:50,366] INFO : Step 36600/50000 rank0 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0699 | Regression_eta: 0.0387 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0439 | LR: 2.50e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9123s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank0 [2026-10-29 21:10:50,604] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 186.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 50.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:15:35,844] INFO : Step 36700/50000 rank0 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0529 | Regression_eta: 0.0292 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0451 | LR: 2.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0054s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank0 [2026-10-29 21:15:36,082] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:20:28,628] INFO : Step 36800/50000 rank0 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0441 | Regression_eta: 0.0277 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0491 | LR: 2.46e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0241s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank0 [2026-10-29 21:20:28,867] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:25:24,424] INFO : Step 36900/50000 rank0 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0261 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0520 | Regression_eta: 0.0389 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0667 | LR: 2.44e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.6773s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank0 [2026-10-29 21:25:24,663] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:30:17,041] INFO : Step 37000/50000 rank0 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0460 | Regression_eta: 0.0358 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0613 | LR: 2.42e-04 | DataLoad Time: 0.0215s | Model Forward Time: 6.6323s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank0 [2026-10-29 21:30:17,282] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:30:17,343] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-37000.pth +rank0 [2026-10-29 21:30:17,642] INFO : Running validation on rank0 +rank0 [2026-10-29 21:30:17,642] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:30:17,642] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION | Step=37000/50000 | Train Loss=0.8679 | Valid Loss=0.8352 | Train Loss Unweighted=0.4039 | Valid Loss Unweighted=0.3939 | Stale=0 | ETA=140.8m +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 21:31:08,662] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari=0.584941 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.70783 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_competing=0.803324 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.666463 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0992138 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.135658 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:31:08,663] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/seconds_per_rank=21.8137 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:31:08,664] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari=0.2965 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding_minus_raw/ari=0.16606 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 21:31:08,665] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:31:08,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:31:08,932] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:31:08,941] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 21:31:35,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:31:36,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:31:37,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:31:39,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:31:49,052] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:31:49,595] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:31:49,599] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:32:05,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:32:07,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:32:08,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:32:10,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:32:20,295] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 21:32:20,306] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:32:20,811] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 21:32:20,815] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 21:32:36,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:32:37,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:32:38,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:32:39,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:32:49,799] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 21:32:49,876] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:32:53,088] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits +rank0 [2026-10-29 21:32:54,133] INFO : Plotted number of elements +rank0 [2026-10-29 21:32:55,676] INFO : Plotted jets +rank0 [2026-10-29 21:32:56,833] INFO : Plotted jet ratio +rank0 [2026-10-29 21:32:56,918] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:33:00,051] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 21:33:01,022] INFO : Plotted number of elements +rank0 [2026-10-29 21:33:02,945] INFO : Plotted jets +rank0 [2026-10-29 21:33:04,064] INFO : Plotted jet ratio +rank0 [2026-10-29 21:33:04,117] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits +rank0 [2026-10-29 21:33:06,396] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits +rank0 [2026-10-29 21:33:07,220] INFO : Plotted number of elements +rank0 [2026-10-29 21:33:08,730] INFO : Plotted jets +rank0 [2026-10-29 21:33:09,914] INFO : Plotted jet ratio +rank0 [2026-10-29 21:37:57,093] INFO : Step 37100/50000 rank0 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.0969 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0440 | Regression_eta: 0.0315 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0512 | LR: 2.40e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.5854s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank0 [2026-10-29 21:37:57,334] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:43:24,477] INFO : Step 37200/50000 rank0 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1157 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0528 | Regression_eta: 0.0335 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0390 | LR: 2.38e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5034s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank0 [2026-10-29 21:43:24,732] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 47.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:48:12,878] INFO : Step 37300/50000 rank0 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0540 | Regression_eta: 0.0294 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0451 | LR: 2.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2818s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank0 [2026-10-29 21:48:13,118] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:52:57,533] INFO : Step 37400/50000 rank0 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0461 | Regression_eta: 0.0330 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0445 | LR: 2.34e-04 | DataLoad Time: 0.0115s | Model Forward Time: 5.2507s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank0 [2026-10-29 21:52:57,770] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:57:44,610] INFO : Step 37500/50000 rank0 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0508 | Regression_eta: 0.0411 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0504 | LR: 2.32e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.4433s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank0 [2026-10-29 21:57:44,847] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:02:30,443] INFO : Step 37600/50000 rank0 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0446 | Regression_eta: 0.0340 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0539 | LR: 2.30e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8506s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank0 [2026-10-29 22:02:30,686] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 188.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 198.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:07:06,322] INFO : Step 37700/50000 rank0 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0218 | Classification_binary: 0.1152 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0547 | Regression_eta: 0.0446 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0490 | LR: 2.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8686s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank0 [2026-10-29 22:07:06,564] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 50.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:11:39,386] INFO : Step 37800/50000 rank0 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0499 | Regression_eta: 0.0375 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0426 | LR: 2.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8674s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank0 [2026-10-29 22:11:39,641] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:16:16,425] INFO : Step 37900/50000 rank0 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0491 | Regression_eta: 0.0407 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0564 | LR: 2.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7363s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank0 [2026-10-29 22:16:16,662] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 183.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:20:57,626] INFO : Step 38000/50000 rank0 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0237 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0423 | Regression_eta: 0.0693 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0522 | LR: 2.22e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9656s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank0 [2026-10-29 22:20:57,874] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 50.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:20:57,897] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-38000.pth +rank0 [2026-10-29 22:20:58,186] INFO : Running validation on rank0 +rank0 [2026-10-29 22:20:58,186] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 22:20:58,186] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 22:21:48,930] INFO : VALIDATION | Step=38000/50000 | Train Loss=0.8316 | Valid Loss=0.8255 | Train Loss Unweighted=0.3880 | Valid Loss Unweighted=0.3913 | Stale=0 | ETA=142.6m +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 22:21:48,931] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari=0.584621 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.728435 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_competing=0.829183 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.657952 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0988936 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.127148 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 22:21:48,932] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/seconds_per_rank=21.5733 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari=0.30011 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 22:21:48,933] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding_minus_raw/ari=0.16967 +rank0 [2026-10-29 22:21:48,934] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 22:21:48,934] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 22:21:48,934] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 22:21:48,938] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:21:49,202] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 22:21:49,213] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 22:22:15,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 22:22:16,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 22:22:17,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 22:22:18,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 22:22:28,846] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 22:22:28,859] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:22:29,426] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 22:22:29,430] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:22:45,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 22:22:46,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 22:22:47,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 22:22:48,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 22:22:58,845] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:22:59,394] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 22:22:59,398] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 22:23:15,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 22:23:16,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 22:23:17,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 22:23:18,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 22:23:28,158] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 22:23:28,173] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits +rank0 [2026-10-29 22:23:31,392] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits +rank0 [2026-10-29 22:23:32,467] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:34,020] INFO : Plotted jets +rank0 [2026-10-29 22:23:35,171] INFO : Plotted jet ratio +rank0 [2026-10-29 22:23:35,243] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:23:38,351] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 22:23:39,341] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:41,250] INFO : Plotted jets +rank0 [2026-10-29 22:23:42,415] INFO : Plotted jet ratio +rank0 [2026-10-29 22:23:42,492] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits +rank0 [2026-10-29 22:23:44,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits +rank0 [2026-10-29 22:23:45,592] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:47,073] INFO : Plotted jets +rank0 [2026-10-29 22:23:48,276] INFO : Plotted jet ratio +rank0 [2026-10-29 22:28:22,446] INFO : Step 38100/50000 rank0 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0266 | Classification_binary: 0.1136 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0448 | Regression_eta: 0.0321 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0389 | LR: 2.20e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1135s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank0 [2026-10-29 22:28:22,700] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:33:07,926] INFO : Step 38200/50000 rank0 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0438 | Regression_eta: 0.0402 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0675 | LR: 2.18e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.4448s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank0 [2026-10-29 22:33:08,174] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:37:42,980] INFO : Step 38300/50000 rank0 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0453 | Regression_eta: 0.0300 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0412 | LR: 2.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.3317s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank0 [2026-10-29 22:37:43,220] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:42:18,265] INFO : Step 38400/50000 rank0 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0554 | Regression_eta: 0.0477 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0532 | LR: 2.14e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1428s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank0 [2026-10-29 22:42:18,505] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 50% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:46:52,954] INFO : Step 38500/50000 rank0 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1105 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0437 | Regression_eta: 0.0368 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0465 | LR: 2.12e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2817s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank0 [2026-10-29 22:46:53,192] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 50% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:51:34,453] INFO : Step 38600/50000 rank0 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0369 | Regression_eta: 0.0356 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0395 | LR: 2.11e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.3882s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank0 [2026-10-29 22:51:34,695] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:56:38,976] INFO : Step 38700/50000 rank0 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0216 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0487 | Regression_eta: 0.0243 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0304 | LR: 2.09e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9405s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank0 [2026-10-29 22:56:39,219] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:02:20,062] INFO : Step 38800/50000 rank0 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1010 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0538 | Regression_eta: 0.0153 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0385 | LR: 2.07e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0395s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank0 [2026-10-29 23:02:20,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:07:30,105] INFO : Step 38900/50000 rank0 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0504 | Regression_eta: 0.0329 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0622 | LR: 2.05e-04 | DataLoad Time: 0.0102s | Model Forward Time: 5.5793s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank0 [2026-10-29 23:07:30,346] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 47.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:12:38,622] INFO : Step 39000/50000 rank0 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1132 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0445 | Regression_eta: 0.0245 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0291 | LR: 2.03e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1148s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank0 [2026-10-29 23:12:38,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:12:38,884] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-39000.pth +rank0 [2026-10-29 23:12:39,293] INFO : Running validation on rank0 +rank0 [2026-10-29 23:12:39,294] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 23:12:39,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 23:13:34,264] INFO : VALIDATION | Step=39000/50000 | Train Loss=0.8109 | Valid Loss=0.8176 | Train Loss Unweighted=0.3771 | Valid Loss Unweighted=0.3805 | Stale=0 | ETA=141.9m +rank0 [2026-10-29 23:13:34,264] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari=0.584824 +rank0 [2026-10-29 23:13:34,265] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.72201 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_competing=0.821336 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.666677 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0990962 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.135872 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-29 23:13:34,266] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/seconds_per_rank=21.8023 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari=0.316957 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding_minus_raw/ari=0.186517 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-29 23:13:34,267] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 23:13:34,273] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:13:34,530] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-29 23:13:34,541] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-29 23:14:06,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 23:14:07,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 23:14:09,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 23:14:10,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 23:14:20,221] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 23:14:20,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:14:20,782] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 23:14:20,785] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:14:36,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 23:14:38,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 23:14:39,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 23:14:40,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 23:14:50,348] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 23:14:50,359] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:14:50,900] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-29 23:14:50,904] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-29 23:15:06,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 23:15:07,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 23:15:08,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 23:15:09,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 23:15:19,244] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 23:15:19,264] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:15:22,468] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits +rank0 [2026-10-29 23:15:23,523] INFO : Plotted number of elements +rank0 [2026-10-29 23:15:25,062] INFO : Plotted jets +rank0 [2026-10-29 23:15:26,234] INFO : Plotted jet ratio +rank0 [2026-10-29 23:15:26,312] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:15:29,820] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits +rank0 [2026-10-29 23:15:30,826] INFO : Plotted number of elements +rank0 [2026-10-29 23:15:32,329] INFO : Plotted jets +rank0 [2026-10-29 23:15:33,462] INFO : Plotted jet ratio +rank0 [2026-10-29 23:15:33,555] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits +rank0 [2026-10-29 23:15:35,860] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits +rank0 [2026-10-29 23:15:36,776] INFO : Plotted number of elements +rank0 [2026-10-29 23:15:38,327] INFO : Plotted jets +rank0 [2026-10-29 23:15:39,509] INFO : Plotted jet ratio +rank0 [2026-10-29 23:20:34,465] INFO : Step 39100/50000 rank0 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0533 | Regression_eta: 0.0265 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0507 | LR: 2.01e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.8786s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank0 [2026-10-29 23:20:34,711] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:25:22,798] INFO : Step 39200/50000 rank0 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0671 | Regression_eta: 0.0388 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0561 | LR: 2.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2424s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank0 [2026-10-29 23:25:23,039] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:29:57,387] INFO : Step 39300/50000 rank0 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0472 | Regression_eta: 0.0386 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0560 | LR: 1.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4267s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank0 [2026-10-29 23:29:57,631] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:34:45,806] INFO : Step 39400/50000 rank0 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1104 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0573 | Regression_eta: 0.0405 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0567 | LR: 1.96e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.4463s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank0 [2026-10-29 23:34:46,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 98% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 5% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 48.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 5% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:39:28,532] INFO : Step 39500/50000 rank0 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0541 | Regression_eta: 0.0279 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0551 | LR: 1.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2462s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank0 [2026-10-29 23:39:28,772] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:43:58,542] INFO : Step 39600/50000 rank0 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0448 | Regression_eta: 0.0282 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0559 | LR: 1.93e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.1308s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank0 [2026-10-29 23:43:58,783] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 50.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:48:43,640] INFO : Step 39700/50000 rank0 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0497 | Regression_eta: 0.0346 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0404 | LR: 1.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0858s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank0 [2026-10-29 23:48:43,880] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:53:55,033] INFO : Step 39800/50000 rank0 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0275 | Classification_binary: 0.0888 | Regression_cos_phi: 0.0307 | Regression_energy: 0.0389 | Regression_eta: 0.0234 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0366 | LR: 1.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2253s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank0 [2026-10-29 23:53:55,273] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:01:07,139] INFO : Step 39900/50000 rank0 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0443 | Regression_eta: 0.0414 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0549 | LR: 1.88e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.5310s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank0 [2026-10-30 00:01:07,378] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:05:56,481] INFO : Step 40000/50000 rank0 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0367 | Regression_eta: 0.0389 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0304 | LR: 1.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5955s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank0 [2026-10-30 00:05:56,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:05:56,756] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-40000.pth +rank0 [2026-10-30 00:05:57,078] INFO : Running validation on rank0 +rank0 [2026-10-30 00:05:57,078] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:05:57,078] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION | Step=40000/50000 | Train Loss=0.8635 | Valid Loss=0.8193 | Train Loss Unweighted=0.4045 | Valid Loss Unweighted=0.3879 | Stale=1 | ETA=139.1m +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 00:06:51,351] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari=0.589757 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.731903 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_competing=0.832487 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.65457 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.10403 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.123765 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:06:51,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/seconds_per_rank=21.7575 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:06:51,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari=0.315377 +rank0 [2026-10-30 00:06:51,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-30 00:06:51,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding_minus_raw/ari=0.184937 +rank0 [2026-10-30 00:06:51,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 00:06:51,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 00:06:51,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:06:51,359] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:06:51,607] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:06:51,617] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 00:07:17,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:07:19,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:07:20,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:07:21,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:07:31,518] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:07:31,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:07:32,019] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:07:32,023] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:07:48,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:07:49,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:07:50,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:07:51,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:08:01,568] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:08:01,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:08:02,099] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 00:08:02,103] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 00:08:17,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 00:08:18,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 00:08:19,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 00:08:20,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 00:08:30,724] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 00:08:30,739] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits +rank0 [2026-10-30 00:08:34,073] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits +rank0 [2026-10-30 00:08:35,178] INFO : Plotted number of elements +rank0 [2026-10-30 00:08:37,128] INFO : Plotted jets +rank0 [2026-10-30 00:08:38,301] INFO : Plotted jet ratio +rank0 [2026-10-30 00:08:38,375] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:08:41,469] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:08:42,469] INFO : Plotted number of elements +rank0 [2026-10-30 00:08:43,968] INFO : Plotted jets +rank0 [2026-10-30 00:08:45,107] INFO : Plotted jet ratio +rank0 [2026-10-30 00:08:45,160] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits +rank0 [2026-10-30 00:08:47,599] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits +rank0 [2026-10-30 00:08:48,504] INFO : Plotted number of elements +rank0 [2026-10-30 00:08:50,080] INFO : Plotted jets +rank0 [2026-10-30 00:08:51,322] INFO : Plotted jet ratio +rank0 [2026-10-30 00:13:51,112] INFO : Step 40100/50000 rank0 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.0927 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0433 | Regression_eta: 0.0212 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0536 | LR: 1.84e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.5520s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank0 [2026-10-30 00:13:51,351] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:18:44,555] INFO : Step 40200/50000 rank0 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1253 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0465 | Regression_eta: 0.0299 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0520 | LR: 1.83e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1109s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank0 [2026-10-30 00:18:44,794] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:23:36,500] INFO : Step 40300/50000 rank0 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0513 | Regression_eta: 0.0357 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0666 | LR: 1.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.3310s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank0 [2026-10-30 00:23:36,740] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:28:39,035] INFO : Step 40400/50000 rank0 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0320 | Regression_energy: 0.0419 | Regression_eta: 0.0316 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0500 | LR: 1.79e-04 | DataLoad Time: 0.0110s | Model Forward Time: 4.9185s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank0 [2026-10-30 00:28:39,277] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:33:25,974] INFO : Step 40500/50000 rank0 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1016 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0436 | Regression_eta: 0.0221 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0235 | LR: 1.78e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0487s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank0 [2026-10-30 00:33:26,213] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 5% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:38:08,528] INFO : Step 40600/50000 rank0 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0438 | Regression_eta: 0.0315 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0388 | LR: 1.76e-04 | DataLoad Time: 0.0137s | Model Forward Time: 4.9944s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank0 [2026-10-30 00:38:08,781] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:42:55,258] INFO : Step 40700/50000 rank0 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0407 | Regression_eta: 0.0307 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0355 | LR: 1.75e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.9298s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank0 [2026-10-30 00:42:55,500] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:47:53,032] INFO : Step 40800/50000 rank0 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1026 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0499 | Regression_eta: 0.0223 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0380 | LR: 1.73e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.5737s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank0 [2026-10-30 00:47:53,281] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:52:47,446] INFO : Step 40900/50000 rank0 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0433 | Regression_eta: 0.0361 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0457 | LR: 1.72e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.3598s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank0 [2026-10-30 00:52:47,695] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:57:39,650] INFO : Step 41000/50000 rank0 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0505 | Regression_eta: 0.0297 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0475 | LR: 1.70e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1086s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank0 [2026-10-30 00:57:39,890] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:57:39,950] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-41000.pth +rank0 [2026-10-30 00:57:40,255] INFO : Running validation on rank0 +rank0 [2026-10-30 00:57:40,255] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:57:40,255] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION | Step=41000/50000 | Train Loss=0.7790 | Valid Loss=0.7996 | Train Loss Unweighted=0.3730 | Valid Loss Unweighted=0.3735 | Stale=0 | ETA=133.5m +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:58:32,270] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari=0.604874 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.73052 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_competing=0.832264 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.660352 +rank0 [2026-10-30 00:58:32,271] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.119146 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.129547 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/seconds_per_rank=21.4183 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 00:58:32,272] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari=0.315577 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding_minus_raw/ari=0.185137 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 00:58:32,273] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:58:32,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:58:32,529] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:58:32,533] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 00:58:59,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:59:00,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:59:01,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:59:02,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:59:12,976] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:59:13,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:59:13,670] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:59:13,674] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 00:59:29,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:59:31,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:59:32,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:59:33,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:59:43,370] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:59:43,381] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:59:43,893] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 00:59:43,897] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 00:59:59,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:00:00,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:00:01,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:00:02,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:00:12,328] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 01:00:12,397] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:00:15,582] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:00:16,655] INFO : Plotted number of elements +rank0 [2026-10-30 01:00:18,578] INFO : Plotted jets +rank0 [2026-10-30 01:00:19,754] INFO : Plotted jet ratio +rank0 [2026-10-30 01:00:19,845] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:00:22,975] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:00:23,983] INFO : Plotted number of elements +rank0 [2026-10-30 01:00:25,488] INFO : Plotted jets +rank0 [2026-10-30 01:00:26,624] INFO : Plotted jet ratio +rank0 [2026-10-30 01:00:26,676] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits +rank0 [2026-10-30 01:00:29,180] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits +rank0 [2026-10-30 01:00:30,068] INFO : Plotted number of elements +rank0 [2026-10-30 01:00:31,597] INFO : Plotted jets +rank0 [2026-10-30 01:00:33,287] INFO : Plotted jet ratio +rank0 [2026-10-30 01:05:11,899] INFO : Step 41100/50000 rank0 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1188 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0609 | Regression_eta: 0.0258 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0372 | LR: 1.69e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.6687s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank0 [2026-10-30 01:05:12,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:10:10,289] INFO : Step 41200/50000 rank0 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0262 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0613 | Regression_eta: 0.0256 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0373 | LR: 1.67e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.2716s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank0 [2026-10-30 01:10:10,528] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:15:01,116] INFO : Step 41300/50000 rank0 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0512 | Regression_eta: 0.0333 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0696 | LR: 1.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0537s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank0 [2026-10-30 01:15:01,359] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:19:56,210] INFO : Step 41400/50000 rank0 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0425 | Regression_eta: 0.0262 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0651 | LR: 1.64e-04 | DataLoad Time: 0.0229s | Model Forward Time: 6.3568s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank0 [2026-10-30 01:19:56,450] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:24:56,393] INFO : Step 41500/50000 rank0 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1062 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0483 | Regression_eta: 0.0288 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0290 | LR: 1.63e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.2405s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank0 [2026-10-30 01:24:56,634] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 3% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 47% +4 8 0x7408, 13395 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 44% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:29:40,501] INFO : Step 41600/50000 rank0 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0414 | Regression_eta: 0.0243 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0637 | LR: 1.61e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.7537s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank0 [2026-10-30 01:29:40,741] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:34:20,624] INFO : Step 41700/50000 rank0 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0426 | Regression_eta: 0.0308 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0376 | LR: 1.60e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3585s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank0 [2026-10-30 01:34:20,866] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:39:11,561] INFO : Step 41800/50000 rank0 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0413 | Regression_eta: 0.0415 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0684 | LR: 1.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5019s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank0 [2026-10-30 01:39:11,800] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:44:07,623] INFO : Step 41900/50000 rank0 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0480 | Regression_eta: 0.0364 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0920 | LR: 1.57e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9507s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank0 [2026-10-30 01:44:07,861] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:49:03,625] INFO : Step 42000/50000 rank0 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0491 | Regression_eta: 0.0378 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0578 | LR: 1.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0458s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank0 [2026-10-30 01:49:03,867] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:49:03,890] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-42000.pth +rank0 [2026-10-30 01:49:04,410] INFO : Running validation on rank0 +rank0 [2026-10-30 01:49:04,410] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 01:49:04,410] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 01:49:57,916] INFO : VALIDATION | Step=42000/50000 | Train Loss=0.8378 | Valid Loss=0.7997 | Train Loss Unweighted=0.3823 | Valid Loss Unweighted=0.3713 | Stale=1 | ETA=125.6m +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 01:49:57,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari=0.585716 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.720263 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_competing=0.81655 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.656062 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0999891 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.125257 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 01:49:57,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/seconds_per_rank=21.8147 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari=0.318444 +rank0 [2026-10-30 01:49:57,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 01:49:57,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding_minus_raw/ari=0.188004 +rank0 [2026-10-30 01:49:57,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 01:49:57,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 01:49:57,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 01:49:57,925] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:49:58,182] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 01:49:58,193] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 01:50:25,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 01:50:26,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 01:50:27,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 01:50:28,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 01:50:38,968] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:50:39,486] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 01:50:39,489] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:50:55,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 01:50:56,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 01:50:57,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 01:50:59,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 01:51:09,573] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-30 01:51:09,584] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:51:10,099] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 01:51:10,102] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 01:51:26,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:51:27,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:51:28,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:51:29,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:51:39,101] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 01:51:39,199] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:51:42,470] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits +rank0 [2026-10-30 01:51:43,564] INFO : Plotted number of elements +rank0 [2026-10-30 01:51:45,112] INFO : Plotted jets +rank0 [2026-10-30 01:51:46,283] INFO : Plotted jet ratio +rank0 [2026-10-30 01:51:46,346] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:51:49,771] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 01:51:50,666] INFO : Plotted number of elements +rank0 [2026-10-30 01:51:52,163] INFO : Plotted jets +rank0 [2026-10-30 01:51:53,292] INFO : Plotted jet ratio +rank0 [2026-10-30 01:51:53,370] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits +rank0 [2026-10-30 01:51:55,808] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits +rank0 [2026-10-30 01:51:56,560] INFO : Plotted number of elements +rank0 [2026-10-30 01:51:58,044] INFO : Plotted jets +rank0 [2026-10-30 01:51:59,240] INFO : Plotted jet ratio +rank0 [2026-10-30 01:57:03,354] INFO : Step 42100/50000 rank0 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1058 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0376 | Regression_eta: 0.0282 | Regression_pt: 0.0373 | Regression_sin_phi: 0.0319 | LR: 1.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.8677s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank0 [2026-10-30 01:57:03,610] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:01:48,074] INFO : Step 42200/50000 rank0 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0273 | Classification_binary: 0.1049 | Regression_cos_phi: 0.0336 | Regression_energy: 0.0357 | Regression_eta: 0.0360 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0438 | LR: 1.53e-04 | DataLoad Time: 0.0177s | Model Forward Time: 7.8771s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank0 [2026-10-30 02:01:48,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 3% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 1% +2 6 0x7408, 15961 49.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +4 8 0x7408, 13395 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 83% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-30 02:06:42,028] INFO : Step 42300/50000 rank0 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0462 | Regression_eta: 0.0272 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0544 | LR: 1.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6153s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank0 [2026-10-30 02:06:42,268] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:11:47,049] INFO : Step 42400/50000 rank0 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0403 | Regression_eta: 0.0317 | Regression_pt: 0.0408 | Regression_sin_phi: 0.0314 | LR: 1.50e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.7828s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank0 [2026-10-30 02:11:47,300] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:16:39,684] INFO : Step 42500/50000 rank0 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0539 | Regression_eta: 0.0265 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0379 | LR: 1.49e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.5590s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank0 [2026-10-30 02:16:39,923] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 7% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 95% +4 8 0x7408, 13395 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 0% +6 10 0x7408, 62036 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 98% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:21:27,848] INFO : Step 42600/50000 rank0 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0464 | Regression_eta: 0.0324 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0549 | LR: 1.48e-04 | DataLoad Time: 0.0198s | Model Forward Time: 7.0909s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank0 [2026-10-30 02:21:28,089] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 55% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 49.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 93% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:26:28,850] INFO : Step 42700/50000 rank0 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0534 | Regression_eta: 0.0282 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0733 | LR: 1.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2736s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank0 [2026-10-30 02:26:29,094] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:31:39,583] INFO : Step 42800/50000 rank0 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0526 | Regression_eta: 0.0285 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0596 | LR: 1.45e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4203s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank0 [2026-10-30 02:31:39,827] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 30% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 93% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:36:26,802] INFO : Step 42900/50000 rank0 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0252 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0325 | Regression_energy: 0.0389 | Regression_eta: 0.0261 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0506 | LR: 1.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4695s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank0 [2026-10-30 02:36:27,059] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:41:14,630] INFO : Step 43000/50000 rank0 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0524 | Regression_eta: 0.0358 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0475 | LR: 1.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3530s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank0 [2026-10-30 02:41:14,876] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:41:14,897] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-43000.pth +rank0 [2026-10-30 02:41:15,304] INFO : Running validation on rank0 +rank0 [2026-10-30 02:41:15,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 02:41:15,305] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 02:42:10,800] INFO : VALIDATION | Step=43000/50000 | Train Loss=0.7601 | Valid Loss=0.7877 | Train Loss Unweighted=0.3546 | Valid Loss Unweighted=0.3682 | Stale=0 | ETA=115.9m +rank0 [2026-10-30 02:42:10,800] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 02:42:10,800] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 02:42:10,800] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari=0.598339 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.720574 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_competing=0.819302 +rank0 [2026-10-30 02:42:10,801] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.657642 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.112612 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.126838 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/seconds_per_rank=21.612 +rank0 [2026-10-30 02:42:10,802] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari=0.334273 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding_minus_raw/ari=0.203833 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 02:42:10,803] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 02:42:10,808] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:42:11,062] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 02:42:11,072] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 02:42:38,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 02:42:39,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 02:42:40,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 02:42:41,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 02:42:51,941] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 02:42:51,953] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:42:52,442] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 02:42:52,445] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 02:43:08,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 02:43:10,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 02:43:11,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 02:43:12,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 02:43:22,349] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:43:23,033] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 02:43:23,037] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 02:43:38,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 02:43:39,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 02:43:40,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 02:43:41,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 02:43:51,956] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 02:43:52,027] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits +rank0 [2026-10-30 02:43:55,282] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits +rank0 [2026-10-30 02:43:56,334] INFO : Plotted number of elements +rank0 [2026-10-30 02:43:57,888] INFO : Plotted jets +rank0 [2026-10-30 02:43:59,474] INFO : Plotted jet ratio +rank0 [2026-10-30 02:43:59,558] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 02:44:02,763] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 02:44:03,738] INFO : Plotted number of elements +rank0 [2026-10-30 02:44:05,252] INFO : Plotted jets +rank0 [2026-10-30 02:44:06,423] INFO : Plotted jet ratio +rank0 [2026-10-30 02:44:06,474] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits +rank0 [2026-10-30 02:44:08,978] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits +rank0 [2026-10-30 02:44:09,838] INFO : Plotted number of elements +rank0 [2026-10-30 02:44:11,358] INFO : Plotted jets +rank0 [2026-10-30 02:44:12,546] INFO : Plotted jet ratio +rank0 [2026-10-30 02:48:52,290] INFO : Step 43100/50000 rank0 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0358 | Regression_eta: 0.0165 | Regression_pt: 0.0364 | Regression_sin_phi: 0.0227 | LR: 1.42e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.8592s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank0 [2026-10-30 02:48:52,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:53:29,845] INFO : Step 43200/50000 rank0 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0313 | Regression_energy: 0.0459 | Regression_eta: 0.0281 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0405 | LR: 1.40e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2360s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank0 [2026-10-30 02:53:30,090] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:58:08,278] INFO : Step 43300/50000 rank0 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0229 | Classification_binary: 0.1256 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0479 | Regression_eta: 0.0232 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0412 | LR: 1.39e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.6972s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank0 [2026-10-30 02:58:08,518] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:02:43,482] INFO : Step 43400/50000 rank0 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0284 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0518 | Regression_eta: 0.0273 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0378 | LR: 1.38e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.1479s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank0 [2026-10-30 03:02:43,724] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:07:17,509] INFO : Step 43500/50000 rank0 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0380 | Regression_eta: 0.0280 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0456 | LR: 1.37e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6458s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank0 [2026-10-30 03:07:17,750] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 41% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +6 10 0x7408, 62036 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:12:00,746] INFO : Step 43600/50000 rank0 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1750 | Regression_cos_phi: 0.1187 | Regression_energy: 0.0441 | Regression_eta: 0.0378 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0973 | LR: 1.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4659s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank0 [2026-10-30 03:12:00,989] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:16:34,396] INFO : Step 43700/50000 rank0 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.0891 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0320 | Regression_eta: 0.0308 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0618 | LR: 1.35e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7882s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank0 [2026-10-30 03:16:34,638] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:21:11,357] INFO : Step 43800/50000 rank0 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0616 | Regression_eta: 0.0210 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0437 | LR: 1.34e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.0133s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank0 [2026-10-30 03:21:11,610] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:26:01,690] INFO : Step 43900/50000 rank0 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0437 | Regression_eta: 0.0384 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0391 | LR: 1.33e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.6276s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank0 [2026-10-30 03:26:01,943] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:30:37,694] INFO : Step 44000/50000 rank0 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0425 | Regression_eta: 0.0331 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0499 | LR: 1.32e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.5781s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank0 [2026-10-30 03:30:37,936] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:30:37,958] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-44000.pth +rank0 [2026-10-30 03:30:38,346] INFO : Running validation on rank0 +rank0 [2026-10-30 03:30:38,346] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 03:30:38,346] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION | Step=44000/50000 | Train Loss=0.7963 | Valid Loss=0.7809 | Train Loss Unweighted=0.3629 | Valid Loss Unweighted=0.3629 | Stale=0 | ETA=103.8m +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 03:31:28,875] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari=0.598001 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.731349 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_competing=0.831851 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.659001 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.112273 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.128197 +rank0 [2026-10-30 03:31:28,876] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/seconds_per_rank=21.5961 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 03:31:28,877] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari=0.318354 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding_minus_raw/ari=0.187915 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 03:31:28,878] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 03:31:28,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:31:29,164] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 03:31:29,169] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 03:31:57,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 03:31:58,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 03:32:00,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 03:32:01,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 03:32:11,380] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 03:32:11,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:32:11,903] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 03:32:11,907] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 03:32:27,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 03:32:28,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 03:32:30,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 03:32:31,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 03:32:41,099] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:32:41,627] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 03:32:41,631] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 03:32:57,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 03:32:58,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 03:32:59,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 03:33:00,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 03:33:10,138] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 03:33:10,171] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits +rank0 [2026-10-30 03:33:13,852] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits +rank0 [2026-10-30 03:33:14,947] INFO : Plotted number of elements +rank0 [2026-10-30 03:33:16,513] INFO : Plotted jets +rank0 [2026-10-30 03:33:17,859] INFO : Plotted jet ratio +rank0 [2026-10-30 03:33:18,014] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 03:33:21,405] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 03:33:22,543] INFO : Plotted number of elements +rank0 [2026-10-30 03:33:24,198] INFO : Plotted jets +rank0 [2026-10-30 03:33:25,522] INFO : Plotted jet ratio +rank0 [2026-10-30 03:33:25,715] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits +rank0 [2026-10-30 03:33:28,373] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits +rank0 [2026-10-30 03:33:29,320] INFO : Plotted number of elements +rank0 [2026-10-30 03:33:30,860] INFO : Plotted jets +rank0 [2026-10-30 03:33:32,561] INFO : Plotted jet ratio +rank0 [2026-10-30 03:38:16,203] INFO : Step 44100/50000 rank0 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1088 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0458 | Regression_eta: 0.0262 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0494 | LR: 1.31e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6169s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank0 [2026-10-30 03:38:16,458] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:43:05,131] INFO : Step 44200/50000 rank0 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0222 | Regression_energy: 0.0459 | Regression_eta: 0.0335 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0345 | LR: 1.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0300s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank0 [2026-10-30 03:43:05,371] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:47:47,070] INFO : Step 44300/50000 rank0 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1580 | Regression_cos_phi: 0.1074 | Regression_energy: 0.0466 | Regression_eta: 0.0373 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0360 | LR: 1.29e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0374s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank0 [2026-10-30 03:47:47,310] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:52:25,406] INFO : Step 44400/50000 rank0 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0430 | Regression_eta: 0.0323 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0411 | LR: 1.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9497s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank0 [2026-10-30 03:52:25,645] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 49.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:57:12,660] INFO : Step 44500/50000 rank0 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1258 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0425 | Regression_eta: 0.0306 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0688 | LR: 1.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0039s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank0 [2026-10-30 03:57:12,900] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:01:48,878] INFO : Step 44600/50000 rank0 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0442 | Regression_eta: 0.0399 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0571 | LR: 1.26e-04 | DataLoad Time: 0.0142s | Model Forward Time: 5.2346s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank0 [2026-10-30 04:01:49,134] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:06:28,904] INFO : Step 44700/50000 rank0 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0231 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0227 | Regression_energy: 0.0420 | Regression_eta: 0.0247 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0488 | LR: 1.25e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.2548s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank0 [2026-10-30 04:06:29,144] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:11:11,715] INFO : Step 44800/50000 rank0 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0451 | Regression_eta: 0.0234 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0309 | LR: 1.24e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.4797s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank0 [2026-10-30 04:11:11,964] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:15:55,188] INFO : Step 44900/50000 rank0 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0530 | Regression_eta: 0.0316 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0410 | LR: 1.23e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3233s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank0 [2026-10-30 04:15:55,443] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:20:36,853] INFO : Step 45000/50000 rank0 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0517 | Regression_eta: 0.0554 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0534 | LR: 1.22e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.5854s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank0 [2026-10-30 04:20:37,107] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:20:37,132] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-45000.pth +rank0 [2026-10-30 04:20:37,556] INFO : Running validation on rank0 +rank0 [2026-10-30 04:20:37,556] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 04:20:37,556] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION | Step=45000/50000 | Train Loss=0.8448 | Valid Loss=0.7810 | Train Loss Unweighted=0.3901 | Valid Loss Unweighted=0.3617 | Stale=1 | ETA=90.1m +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 04:21:28,530] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari=0.60025 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.740743 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_competing=0.843224 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.659273 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.114523 +rank0 [2026-10-30 04:21:28,531] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.128469 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/seconds_per_rank=22.0223 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 04:21:28,532] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari=0.332781 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding_minus_raw/ari=0.202342 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 04:21:28,533] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 04:21:28,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:21:28,783] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 04:21:28,793] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 04:21:56,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 04:21:57,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 04:21:58,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 04:22:00,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 04:22:10,115] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:22:10,892] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 04:22:10,896] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 04:22:27,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 04:22:28,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 04:22:29,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 04:22:30,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 04:22:40,506] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:22:41,058] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 04:22:41,062] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 04:22:56,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 04:22:57,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 04:22:58,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 04:22:59,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 04:23:09,676] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 04:23:09,723] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits +rank0 [2026-10-30 04:23:12,963] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits +rank0 [2026-10-30 04:23:14,011] INFO : Plotted number of elements +rank0 [2026-10-30 04:23:15,578] INFO : Plotted jets +rank0 [2026-10-30 04:23:17,166] INFO : Plotted jet ratio +rank0 [2026-10-30 04:23:17,268] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 04:23:20,531] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 04:23:21,636] INFO : Plotted number of elements +rank0 [2026-10-30 04:23:23,248] INFO : Plotted jets +rank0 [2026-10-30 04:23:24,462] INFO : Plotted jet ratio +rank0 [2026-10-30 04:23:24,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits +rank0 [2026-10-30 04:23:27,082] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits +rank0 [2026-10-30 04:23:27,954] INFO : Plotted number of elements +rank0 [2026-10-30 04:23:29,437] INFO : Plotted jets +rank0 [2026-10-30 04:23:30,650] INFO : Plotted jet ratio +rank0 [2026-10-30 04:28:05,906] INFO : Step 45100/50000 rank0 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.0900 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0483 | Regression_eta: 0.0245 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0309 | LR: 1.21e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1780s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank0 [2026-10-30 04:28:06,162] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:32:58,159] INFO : Step 45200/50000 rank0 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0425 | Regression_eta: 0.0234 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0587 | LR: 1.20e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1739s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank0 [2026-10-30 04:32:58,406] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:37:52,762] INFO : Step 45300/50000 rank0 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0276 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0295 | Regression_energy: 0.0262 | Regression_eta: 0.0205 | Regression_pt: 0.0276 | Regression_sin_phi: 0.0235 | LR: 1.19e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.4438s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank0 [2026-10-30 04:37:53,021] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:42:39,071] INFO : Step 45400/50000 rank0 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1002 | Regression_cos_phi: 0.0265 | Regression_energy: 0.0402 | Regression_eta: 0.0309 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0528 | LR: 1.19e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0467s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank0 [2026-10-30 04:42:39,311] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:47:26,993] INFO : Step 45500/50000 rank0 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1127 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0398 | Regression_eta: 0.0318 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0284 | LR: 1.18e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9962s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank0 [2026-10-30 04:47:27,232] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:52:17,034] INFO : Step 45600/50000 rank0 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0434 | Regression_eta: 0.0464 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0563 | LR: 1.17e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2301s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank0 [2026-10-30 04:52:17,287] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:57:08,486] INFO : Step 45700/50000 rank0 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0469 | Regression_eta: 0.0260 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0662 | LR: 1.16e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.1281s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank0 [2026-10-30 04:57:08,746] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 53% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:01:56,860] INFO : Step 45800/50000 rank0 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0433 | Regression_eta: 0.0250 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0420 | LR: 1.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4249s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank0 [2026-10-30 05:01:57,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 56% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:06:42,682] INFO : Step 45900/50000 rank0 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0501 | Regression_eta: 0.0252 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0408 | LR: 1.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9063s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank0 [2026-10-30 05:06:42,923] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 59% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:11:36,778] INFO : Step 46000/50000 rank0 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1013 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0421 | Regression_eta: 0.0275 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0409 | LR: 1.14e-04 | DataLoad Time: 0.0223s | Model Forward Time: 7.1693s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank0 [2026-10-30 05:11:37,035] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 59% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:11:37,057] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-46000.pth +rank0 [2026-10-30 05:11:37,353] INFO : Running validation on rank0 +rank0 [2026-10-30 05:11:37,354] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 05:11:37,354] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION | Step=46000/50000 | Train Loss=0.7614 | Valid Loss=0.7854 | Train Loss Unweighted=0.3547 | Valid Loss Unweighted=0.3650 | Stale=2 | ETA=75.0m +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 05:12:28,004] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari=0.59537 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.732694 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_competing=0.833843 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.661256 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.109642 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.130451 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 05:12:28,005] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/seconds_per_rank=21.5661 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 05:12:28,006] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari=0.309683 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding_minus_raw/ari=0.179243 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 05:12:28,007] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 05:12:28,012] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:12:28,260] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 05:12:28,264] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 05:12:53,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 05:12:54,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 05:12:55,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 05:12:56,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 05:13:06,619] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 05:13:06,686] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:13:07,159] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 05:13:07,163] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 05:13:23,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 05:13:24,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 05:13:25,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 05:13:26,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 05:13:36,703] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 05:13:36,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:13:37,282] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 05:13:37,286] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 05:13:52,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 05:13:53,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 05:13:54,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 05:13:55,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 05:14:05,893] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 05:14:05,908] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits +rank0 [2026-10-30 05:14:09,170] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits +rank0 [2026-10-30 05:14:10,271] INFO : Plotted number of elements +rank0 [2026-10-30 05:14:11,832] INFO : Plotted jets +rank0 [2026-10-30 05:14:13,018] INFO : Plotted jet ratio +rank0 [2026-10-30 05:14:13,077] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 05:14:16,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 05:14:17,291] INFO : Plotted number of elements +rank0 [2026-10-30 05:14:18,812] INFO : Plotted jets +rank0 [2026-10-30 05:14:19,962] INFO : Plotted jet ratio +rank0 [2026-10-30 05:14:20,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits +rank0 [2026-10-30 05:14:22,518] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits +rank0 [2026-10-30 05:14:23,388] INFO : Plotted number of elements +rank0 [2026-10-30 05:14:25,280] INFO : Plotted jets +rank0 [2026-10-30 05:14:26,494] INFO : Plotted jet ratio +rank0 [2026-10-30 05:19:01,193] INFO : Step 46100/50000 rank0 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0285 | Regression_energy: 0.0548 | Regression_eta: 0.0269 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0431 | LR: 1.13e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.2453s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank0 [2026-10-30 05:19:01,432] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:23:35,107] INFO : Step 46200/50000 rank0 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0314 | Regression_energy: 0.0400 | Regression_eta: 0.0283 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0397 | LR: 1.13e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4533s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank0 [2026-10-30 05:23:35,347] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:28:09,285] INFO : Step 46300/50000 rank0 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0417 | Regression_eta: 0.0249 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0324 | LR: 1.12e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.6535s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank0 [2026-10-30 05:28:09,524] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:32:47,316] INFO : Step 46400/50000 rank0 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0362 | Regression_eta: 0.0306 | Regression_pt: 0.0368 | Regression_sin_phi: 0.0558 | LR: 1.11e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4568s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank0 [2026-10-30 05:32:47,563] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:37:22,479] INFO : Step 46500/50000 rank0 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1140 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0462 | Regression_eta: 0.0383 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0396 | LR: 1.11e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.4873s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank0 [2026-10-30 05:37:22,730] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:41:58,105] INFO : Step 46600/50000 rank0 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0418 | Regression_eta: 0.0228 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0342 | LR: 1.10e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5462s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank0 [2026-10-30 05:41:58,357] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:46:33,096] INFO : Step 46700/50000 rank0 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1057 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0496 | Regression_eta: 0.0274 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0353 | LR: 1.10e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3367s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank0 [2026-10-30 05:46:33,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:51:05,902] INFO : Step 46800/50000 rank0 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1032 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0388 | Regression_eta: 0.0421 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0413 | LR: 1.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.4622s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank0 [2026-10-30 05:51:06,148] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:55:42,229] INFO : Step 46900/50000 rank0 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0461 | Regression_eta: 0.0277 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0524 | LR: 1.09e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3827s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank0 [2026-10-30 05:55:42,493] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:00:21,493] INFO : Step 47000/50000 rank0 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0450 | Regression_eta: 0.0282 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0407 | LR: 1.08e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.6011s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank0 [2026-10-30 06:00:21,748] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:00:21,769] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-47000.pth +rank0 [2026-10-30 06:00:22,074] INFO : Running validation on rank0 +rank0 [2026-10-30 06:00:22,075] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 06:00:22,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION | Step=47000/50000 | Train Loss=0.8023 | Valid Loss=0.7714 | Train Loss Unweighted=0.3632 | Valid Loss Unweighted=0.3533 | Stale=0 | ETA=58.1m +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 06:01:12,842] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari=0.59383 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.743828 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_competing=0.846933 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.674533 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.108103 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.143728 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 06:01:12,843] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/seconds_per_rank=21.5734 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 06:01:12,844] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari=0.316006 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding_minus_raw/ari=0.185566 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 06:01:12,845] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 06:01:12,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:01:13,099] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 06:01:13,103] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 06:01:37,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 06:01:38,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 06:01:39,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 06:01:41,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 06:01:51,115] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:01:51,182] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:01:51,692] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 06:01:51,696] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:02:07,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 06:02:09,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 06:02:10,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 06:02:11,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 06:02:21,374] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:02:21,385] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:02:21,870] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 06:02:21,874] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 06:02:37,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 06:02:38,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 06:02:39,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 06:02:40,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 06:02:50,696] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 06:02:50,819] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits +rank0 [2026-10-30 06:02:54,048] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits +rank0 [2026-10-30 06:02:55,093] INFO : Plotted number of elements +rank0 [2026-10-30 06:02:56,649] INFO : Plotted jets +rank0 [2026-10-30 06:02:57,820] INFO : Plotted jet ratio +rank0 [2026-10-30 06:02:57,870] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:03:01,020] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:03:02,483] INFO : Plotted number of elements +rank0 [2026-10-30 06:03:03,991] INFO : Plotted jets +rank0 [2026-10-30 06:03:05,145] INFO : Plotted jet ratio +rank0 [2026-10-30 06:03:05,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits +rank0 [2026-10-30 06:03:07,520] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits +rank0 [2026-10-30 06:03:08,340] INFO : Plotted number of elements +rank0 [2026-10-30 06:03:09,853] INFO : Plotted jets +rank0 [2026-10-30 06:03:11,064] INFO : Plotted jet ratio +rank0 [2026-10-30 06:07:51,863] INFO : Step 47100/50000 rank0 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.0893 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0383 | Regression_eta: 0.0234 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0277 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2920s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank0 [2026-10-30 06:07:52,114] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:12:30,298] INFO : Step 47200/50000 rank0 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1047 | Regression_cos_phi: 0.0289 | Regression_energy: 0.0547 | Regression_eta: 0.0229 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0438 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.6513s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank0 [2026-10-30 06:12:30,542] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:17:04,765] INFO : Step 47300/50000 rank0 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0247 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0304 | Regression_energy: 0.0403 | Regression_eta: 0.0227 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0410 | LR: 1.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6459s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank0 [2026-10-30 06:17:05,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:21:39,218] INFO : Step 47400/50000 rank0 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.0896 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0411 | Regression_eta: 0.0214 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0561 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2222s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank0 [2026-10-30 06:21:39,472] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:26:16,511] INFO : Step 47500/50000 rank0 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1053 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0372 | Regression_eta: 0.0226 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0595 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3862s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank0 [2026-10-30 06:26:16,760] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:30:50,923] INFO : Step 47600/50000 rank0 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0447 | Regression_eta: 0.0386 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0540 | LR: 1.05e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.7533s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank0 [2026-10-30 06:30:51,174] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:35:26,200] INFO : Step 47700/50000 rank0 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0441 | Regression_eta: 0.0312 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0712 | LR: 1.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5312s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank0 [2026-10-30 06:35:26,449] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:40:01,238] INFO : Step 47800/50000 rank0 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0258 | Classification_binary: 0.1114 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0467 | Regression_eta: 0.0271 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0410 | LR: 1.04e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.6094s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank0 [2026-10-30 06:40:01,486] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:44:37,165] INFO : Step 47900/50000 rank0 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0454 | Regression_eta: 0.0241 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0516 | LR: 1.04e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4758s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank0 [2026-10-30 06:44:37,410] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:49:10,397] INFO : Step 48000/50000 rank0 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0409 | Regression_eta: 0.0415 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0599 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.6420s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank0 [2026-10-30 06:49:10,649] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:49:10,745] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-48000.pth +rank0 [2026-10-30 06:49:11,040] INFO : Running validation on rank0 +rank0 [2026-10-30 06:49:11,040] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 06:49:11,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 06:50:01,892] INFO : VALIDATION | Step=48000/50000 | Train Loss=0.7937 | Valid Loss=0.7642 | Train Loss Unweighted=0.3728 | Valid Loss Unweighted=0.3530 | Stale=0 | ETA=40.0m +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 06:50:01,893] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari=0.591679 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.73887 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_competing=0.841546 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.67946 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.105951 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.148655 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 06:50:01,894] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/seconds_per_rank=21.631 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:50:01,895] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari=0.313517 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding_minus_raw/ari=0.183077 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 06:50:01,896] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 06:50:01,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:50:02,160] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 06:50:02,163] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 06:50:27,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 06:50:28,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 06:50:29,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 06:50:31,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 06:50:41,252] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:50:41,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:50:41,779] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 06:50:41,783] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:50:57,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 06:50:58,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 06:50:59,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 06:51:00,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 06:51:10,914] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:51:11,430] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 06:51:11,434] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 06:51:27,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 06:51:28,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 06:51:29,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 06:51:30,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 06:51:40,678] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 06:51:40,694] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits +rank0 [2026-10-30 06:51:43,923] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits +rank0 [2026-10-30 06:51:44,975] INFO : Plotted number of elements +rank0 [2026-10-30 06:51:46,513] INFO : Plotted jets +rank0 [2026-10-30 06:51:47,701] INFO : Plotted jet ratio +rank0 [2026-10-30 06:51:47,775] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:51:50,938] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 06:51:51,954] INFO : Plotted number of elements +rank0 [2026-10-30 06:51:53,475] INFO : Plotted jets +rank0 [2026-10-30 06:51:54,650] INFO : Plotted jet ratio +rank0 [2026-10-30 06:51:54,700] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits +rank0 [2026-10-30 06:51:57,175] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits +rank0 [2026-10-30 06:51:58,037] INFO : Plotted number of elements +rank0 [2026-10-30 06:52:00,045] INFO : Plotted jets +rank0 [2026-10-30 06:52:01,189] INFO : Plotted jet ratio +rank0 [2026-10-30 06:56:35,746] INFO : Step 48100/50000 rank0 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0389 | Regression_eta: 0.0341 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0586 | LR: 1.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2759s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank0 [2026-10-30 06:56:36,001] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:01:17,102] INFO : Step 48200/50000 rank0 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1035 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0386 | Regression_eta: 0.0260 | Regression_pt: 0.0399 | Regression_sin_phi: 0.0544 | LR: 1.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.9636s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank0 [2026-10-30 07:01:17,355] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:05:54,106] INFO : Step 48300/50000 rank0 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0418 | Regression_eta: 0.0403 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0557 | LR: 1.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.7590s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank0 [2026-10-30 07:05:54,363] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:10:31,592] INFO : Step 48400/50000 rank0 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1334 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0400 | Regression_eta: 0.0344 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0450 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6551s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank0 [2026-10-30 07:10:31,853] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +4 8 0x7408, 13395 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:15:12,054] INFO : Step 48500/50000 rank0 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0268 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0438 | Regression_eta: 0.0272 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0342 | LR: 1.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.5921s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank0 [2026-10-30 07:15:12,313] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:19:53,308] INFO : Step 48600/50000 rank0 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0373 | Regression_eta: 0.0318 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0500 | LR: 1.02e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.5785s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank0 [2026-10-30 07:19:53,557] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:24:30,703] INFO : Step 48700/50000 rank0 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0506 | Regression_eta: 0.0285 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0525 | LR: 1.02e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.6409s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank0 [2026-10-30 07:24:30,991] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:29:11,000] INFO : Step 48800/50000 rank0 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.0842 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0485 | Regression_eta: 0.0214 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0413 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.7048s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank0 [2026-10-30 07:29:11,262] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +4 8 0x7408, 13395 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:33:48,967] INFO : Step 48900/50000 rank0 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0268 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0244 | Regression_energy: 0.0391 | Regression_eta: 0.0262 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0361 | LR: 1.01e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4606s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank0 [2026-10-30 07:33:49,214] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:38:23,446] INFO : Step 49000/50000 rank0 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0503 | Regression_eta: 0.0293 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0493 | LR: 1.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6453s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank0 [2026-10-30 07:38:23,695] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +4 8 0x7408, 13395 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:38:23,719] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-49000.pth +rank0 [2026-10-30 07:38:24,023] INFO : Running validation on rank0 +rank0 [2026-10-30 07:38:24,023] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 07:38:24,023] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 07:39:14,589] INFO : VALIDATION | Step=49000/50000 | Train Loss=0.8118 | Valid Loss=0.7793 | Train Loss Unweighted=0.3751 | Valid Loss Unweighted=0.3564 | Stale=1 | ETA=20.6m +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 07:39:14,590] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari=0.584329 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.742709 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_competing=0.846107 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.668503 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0986012 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.137698 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 07:39:14,591] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/seconds_per_rank=21.7009 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 07:39:14,592] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari=0.315141 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding_minus_raw/ari=0.184701 +rank0 [2026-10-30 07:39:14,593] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 07:39:14,594] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 07:39:14,594] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 07:39:14,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:39:14,868] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 07:39:14,872] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 07:39:39,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 07:39:40,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 07:39:41,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 07:39:43,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 07:39:53,244] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 07:39:53,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:39:53,779] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 07:39:53,783] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 07:40:10,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 07:40:11,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 07:40:12,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 07:40:13,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 07:40:23,492] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 07:40:23,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:40:24,012] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 07:40:24,016] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 07:40:39,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 07:40:40,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 07:40:41,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 07:40:42,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 07:40:52,801] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 07:40:52,907] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits +rank0 [2026-10-30 07:40:56,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits +rank0 [2026-10-30 07:40:57,172] INFO : Plotted number of elements +rank0 [2026-10-30 07:40:59,181] INFO : Plotted jets +rank0 [2026-10-30 07:41:00,378] INFO : Plotted jet ratio +rank0 [2026-10-30 07:41:00,428] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 07:41:03,575] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 07:41:04,601] INFO : Plotted number of elements +rank0 [2026-10-30 07:41:06,116] INFO : Plotted jets +rank0 [2026-10-30 07:41:07,267] INFO : Plotted jet ratio +rank0 [2026-10-30 07:41:07,322] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits +rank0 [2026-10-30 07:41:09,816] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits +rank0 [2026-10-30 07:41:10,690] INFO : Plotted number of elements +rank0 [2026-10-30 07:41:12,207] INFO : Plotted jets +rank0 [2026-10-30 07:41:13,413] INFO : Plotted jet ratio +rank0 [2026-10-30 07:45:46,931] INFO : Step 49100/50000 rank0 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.0953 | Regression_cos_phi: 0.0229 | Regression_energy: 0.0416 | Regression_eta: 0.0229 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0271 | LR: 1.01e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.3760s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank0 [2026-10-30 07:45:47,181] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:50:31,213] INFO : Step 49200/50000 rank0 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.0952 | Regression_cos_phi: 0.0301 | Regression_energy: 0.0581 | Regression_eta: 0.0267 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0350 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5149s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank0 [2026-10-30 07:50:31,469] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:55:08,104] INFO : Step 49300/50000 rank0 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1178 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0385 | Regression_eta: 0.0336 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0359 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5995s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank0 [2026-10-30 07:55:08,344] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:59:44,207] INFO : Step 49400/50000 rank0 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0404 | Regression_eta: 0.0223 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0323 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.7450s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank0 [2026-10-30 07:59:44,487] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:04:25,436] INFO : Step 49500/50000 rank0 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1149 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0375 | Regression_eta: 0.0198 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0381 | LR: 1.00e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0133s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank0 [2026-10-30 08:04:25,685] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:08:59,674] INFO : Step 49600/50000 rank0 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0422 | Regression_eta: 0.0406 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0579 | LR: 1.00e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.6271s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank0 [2026-10-30 08:08:59,931] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:13:37,596] INFO : Step 49700/50000 rank0 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1190 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0466 | Regression_eta: 0.0343 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0642 | LR: 1.00e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.3690s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank0 [2026-10-30 08:13:37,837] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:18:12,192] INFO : Step 49800/50000 rank0 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0260 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0478 | Regression_eta: 0.0265 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0442 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3018s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank0 [2026-10-30 08:18:12,451] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:22:54,109] INFO : Step 49900/50000 rank0 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0372 | Regression_eta: 0.0184 | Regression_pt: 0.0381 | Regression_sin_phi: 0.0298 | LR: 1.00e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.8930s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank0 [2026-10-30 08:22:54,365] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 47.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:27:37,929] INFO : Step 50000/50000 rank0 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0486 | Regression_eta: 0.0415 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0524 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.0237s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank0 [2026-10-30 08:27:38,175] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 08:27:38,208] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-50000.pth +rank0 [2026-10-30 08:27:38,533] INFO : Running validation on rank0 +rank0 [2026-10-30 08:27:38,533] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 08:27:38,533] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION | Step=50000/50000 | Train Loss=0.8463 | Valid Loss=0.7854 | Train Loss Unweighted=0.3713 | Valid Loss Unweighted=0.3564 | Stale=2 | ETA=0.0m +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 08:28:29,743] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/competing_candidates=0.766707 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.269697 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.934031 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.939037 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/target_layer_exists=0.984975 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.831526 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_competing=0.954483 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.5891 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari=0.612845 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.745501 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_competing=0.849223 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.673056 +rank0 [2026-10-30 08:28:29,744] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.127117 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.142251 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_triplet_samples=63846 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_triplet_samples=202077 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/oracle_graph/ari=0.972219 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari=0.485727 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari_threshold=0.65 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.652934 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_competing=0.733459 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/triplet_accuracy=0.530805 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/seconds_per_rank=21.655 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.834561 +rank0 [2026-10-30 08:28:29,745] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.957821 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.605553 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari=0.30646 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding_minus_raw/ari=0.176021 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/oracle_graph/ari=0.905269 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari=0.13044 +rank0 [2026-10-30 08:28:29,746] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 08:28:29,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 08:28:30,020] INFO : test_dataset: clic_edm_ttbar_hits, 510 +rank0 [2026-10-30 08:28:30,035] INFO : Running predictions on clic_edm_ttbar_hits +rank0 [2026-10-30 08:29:00,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 08:29:01,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 08:29:02,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 08:29:03,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 08:29:13,855] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 08:29:13,910] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 08:29:14,402] INFO : test_dataset: clic_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 08:29:14,406] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank0 [2026-10-30 08:29:30,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 08:29:31,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 08:29:33,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 08:29:35,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 08:29:45,551] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 08:29:46,062] INFO : test_dataset: clic_edm_qq_hits, 510 +rank0 [2026-10-30 08:29:46,066] INFO : Running predictions on clic_edm_qq_hits +rank0 [2026-10-30 08:30:01,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 08:30:02,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 08:30:03,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 08:30:04,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 08:30:14,813] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 08:30:14,830] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits +rank0 [2026-10-30 08:30:18,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits +rank0 [2026-10-30 08:30:19,346] INFO : Plotted number of elements +rank0 [2026-10-30 08:30:21,015] INFO : Plotted jets +rank0 [2026-10-30 08:30:22,187] INFO : Plotted jet ratio +rank0 [2026-10-30 08:30:22,237] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 08:30:25,393] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits +rank0 [2026-10-30 08:30:26,407] INFO : Plotted number of elements +rank0 [2026-10-30 08:30:27,926] INFO : Plotted jets +rank0 [2026-10-30 08:30:29,067] INFO : Plotted jet ratio +rank0 [2026-10-30 08:30:29,121] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits +rank0 [2026-10-30 08:30:31,592] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits +rank0 [2026-10-30 08:30:32,458] INFO : Plotted number of elements +rank0 [2026-10-30 08:30:34,462] INFO : Plotted jets +rank0 [2026-10-30 08:30:35,667] INFO : Plotted jet ratio +rank0 [2026-10-30 08:30:36,180] INFO : Training completed. Total time on device 0: 1060.268min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.1 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..2cda479d394d5581b83f8522823bee6ffb408283 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.1 @@ -0,0 +1,2654 @@ +rank1 [2026-10-28 11:12:32,414] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-28 11:12:32,415] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank1 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank1 [2026-10-28 11:12:57,210] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-28 11:12:57,210] INFO : Initializing backbone layer 0 +rank1 [2026-10-28 11:12:57,248] INFO : Initializing backbone layer 1 +rank1 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 2 +rank1 [2026-10-28 11:12:57,277] INFO : Initializing backbone layer 3 +rank1 [2026-10-28 11:12:57,291] INFO : Initializing backbone layer 4 +rank1 [2026-10-28 11:12:57,306] INFO : Initializing backbone layer 5 +rank1 [2026-10-28 11:12:57,320] INFO : Backbone initialization took 0.11s +rank1 [2026-10-28 11:12:57,321] INFO : backbone parameters: 7901376 +rank1 [2026-10-28 11:12:57,321] INFO : Initializing output DNNs +rank1 [2026-10-28 11:12:57,384] INFO : Output DNNs initialization took 0.06s +rank1 [2026-10-28 11:12:57,384] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-28 11:12:57,384] INFO : jagged_attention enabled=False +rank1 [2026-10-28 11:12:57,384] INFO : flash_attn_varlen enabled=False +rank1 [2026-10-28 11:12:57,384] INFO : task_queries enabled=False +rank1 [2026-10-28 11:12:57,384] INFO : classification_norm parameters: 0 +rank1 [2026-10-28 11:12:57,384] INFO : regression_norm parameters: 0 +rank1 [2026-10-28 11:12:57,384] INFO : classification_query parameters: 0 +rank1 [2026-10-28 11:12:57,384] INFO : regression_query parameters: 0 +rank1 [2026-10-28 11:12:57,384] INFO : classification_readout parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : regression_readout parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : output_mode=set +rank1 [2026-10-28 11:12:57,385] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-28 11:12:57,385] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : nn_pid parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : nn_pt parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : nn_eta parameters: 0 +rank1 [2026-10-28 11:12:57,385] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-28 11:12:57,386] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-28 11:12:57,386] INFO : nn_energy parameters: 0 +rank1 [2026-10-28 11:12:57,387] INFO : Total MLPF parameters: 12786964 +rank1 [2026-10-28 11:12:57,387] INFO : MLPF __init__ done +rank1 [2026-10-28 11:12:57,387] INFO : Instantiated model +rank1 [2026-10-28 11:12:57,387] INFO : Moving model to device rank=1 +rank1 [2026-10-28 11:12:57,418] INFO : Moved model to device rank=1 +rank1 [2026-10-28 11:12:57,421] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-28 11:12:57,422] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-28 11:12:57,715] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-28 11:12:57,921] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank1 [2026-10-28 11:12:57,921] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-28 11:12:57,922] INFO : Trainable parameters: 12786964 +rank1 [2026-10-28 11:12:57,922] INFO : Non-trainable parameters: 23268 +rank1 [2026-10-28 11:12:57,922] INFO : Total parameters: 12810232 +rank1 [2026-10-28 11:12:57,925] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-28 11:12:57,925] INFO : Getting dataloaders +rank1 [2026-10-28 11:12:57,925] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:03,688] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:04,774] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,273] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,531] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,791] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-28 11:13:06,791] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-28 11:13:20,956] INFO : Got dataloaders +rank1 [2026-10-28 11:13:20,956] INFO : loader split=train rank=1 len=21108 +rank1 [2026-10-28 11:13:20,956] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-28 11:13:20,956] INFO : Starting training from step 1 to 50000 on rank 1 of 8 +rank1 [2026-10-28 11:13:20,956] INFO : Creating train_iterator +rank1 [2026-10-28 11:13:20,956] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-28 11:13:35,226] INFO : Created train_iterator +rank1 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank1 [2026-10-28 11:18:16,510] INFO : Step 100/50000 rank1 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7417 | Classification_binary: 0.4279 | Regression_cos_phi: 0.0063 | Regression_energy: 0.7347 | Regression_eta: 0.0134 | Regression_pt: 0.6933 | Regression_sin_phi: 0.0061 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.6167s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank1 [2026-10-28 11:22:47,963] INFO : Step 200/50000 rank1 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3515 | Classification_binary: 0.5476 | Regression_cos_phi: 0.3326 | Regression_energy: 0.3482 | Regression_eta: 0.2885 | Regression_pt: 0.3542 | Regression_sin_phi: 0.3732 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.4378s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank1 [2026-10-28 11:27:17,891] INFO : Step 300/50000 rank1 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3242 | Classification_binary: 0.4328 | Regression_cos_phi: 0.4190 | Regression_energy: 0.3781 | Regression_eta: 0.3472 | Regression_pt: 0.3722 | Regression_sin_phi: 0.3511 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.3673s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank1 [2026-10-28 11:31:48,894] INFO : Step 400/50000 rank1 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2497 | Classification_binary: 0.4304 | Regression_cos_phi: 0.2493 | Regression_energy: 0.3756 | Regression_eta: 0.2164 | Regression_pt: 0.3789 | Regression_sin_phi: 0.3000 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.2447s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank1 [2026-10-28 11:36:20,164] INFO : Step 500/50000 rank1 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3137 | Classification_binary: 0.5169 | Regression_cos_phi: 0.3108 | Regression_energy: 0.2721 | Regression_eta: 0.2822 | Regression_pt: 0.2849 | Regression_sin_phi: 0.3337 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 4.4689s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank1 [2026-10-28 11:40:54,777] INFO : Step 600/50000 rank1 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2282 | Classification_binary: 0.4888 | Regression_cos_phi: 0.2766 | Regression_energy: 0.3150 | Regression_eta: 0.1750 | Regression_pt: 0.3222 | Regression_sin_phi: 0.2279 | LR: 1.00e-03 | DataLoad Time: 0.0136s | Model Forward Time: 4.5149s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank1 [2026-10-28 11:45:32,573] INFO : Step 700/50000 rank1 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2153 | Classification_binary: 0.4338 | Regression_cos_phi: 0.2625 | Regression_energy: 0.2726 | Regression_eta: 0.1921 | Regression_pt: 0.2837 | Regression_sin_phi: 0.2492 | LR: 1.00e-03 | DataLoad Time: 0.0174s | Model Forward Time: 3.7409s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank1 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank1 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2916 | Classification_binary: 0.4344 | Regression_cos_phi: 0.2537 | Regression_energy: 0.3090 | Regression_eta: 0.2094 | Regression_pt: 0.3161 | Regression_sin_phi: 0.3270 | LR: 9.99e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.2804s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank1 [2026-10-28 11:54:58,535] INFO : Step 900/50000 rank1 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2478 | Classification_binary: 0.4912 | Regression_cos_phi: 0.3041 | Regression_energy: 0.2631 | Regression_eta: 0.1781 | Regression_pt: 0.2766 | Regression_sin_phi: 0.2343 | LR: 9.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4556s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank1 [2026-10-28 11:59:55,247] INFO : Step 1000/50000 rank1 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1999 | Classification_binary: 0.4515 | Regression_cos_phi: 0.2158 | Regression_energy: 0.2310 | Regression_eta: 0.1498 | Regression_pt: 0.2392 | Regression_sin_phi: 0.2100 | LR: 9.99e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.7551s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank1 [2026-10-28 11:59:56,331] INFO : Running validation on rank1 +rank1 [2026-10-28 11:59:56,331] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 12:00:48,695] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:00:49,300] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 12:01:16,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:01:17,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:01:18,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:01:19,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:01:29,915] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:01:30,838] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 12:01:46,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:01:47,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:01:48,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:01:49,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:01:59,361] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:02:00,465] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:02:01,008] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 12:02:16,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:02:17,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:02:18,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:02:19,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:02:29,042] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 12:07:33,551] INFO : Step 1100/50000 rank1 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1834 | Classification_binary: 0.3462 | Regression_cos_phi: 0.2671 | Regression_energy: 0.2492 | Regression_eta: 0.1640 | Regression_pt: 0.2655 | Regression_sin_phi: 0.3416 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8139s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank1 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank1 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2406 | Classification_binary: 0.4748 | Regression_cos_phi: 0.2476 | Regression_energy: 0.2674 | Regression_eta: 0.2192 | Regression_pt: 0.2770 | Regression_sin_phi: 0.2792 | LR: 9.99e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4375s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank1 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank1 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2239 | Classification_binary: 0.3805 | Regression_cos_phi: 0.2405 | Regression_energy: 0.2422 | Regression_eta: 0.1767 | Regression_pt: 0.2525 | Regression_sin_phi: 0.2240 | LR: 9.98e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2250s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank1 [2026-10-28 12:21:55,309] INFO : Step 1400/50000 rank1 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1843 | Classification_binary: 0.3443 | Regression_cos_phi: 0.3301 | Regression_energy: 0.2438 | Regression_eta: 0.2340 | Regression_pt: 0.2462 | Regression_sin_phi: 0.3100 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5549s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank1 [2026-10-28 12:26:33,581] INFO : Step 1500/50000 rank1 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2276 | Classification_binary: 0.4098 | Regression_cos_phi: 0.2044 | Regression_energy: 0.2205 | Regression_eta: 0.2018 | Regression_pt: 0.2288 | Regression_sin_phi: 0.2897 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9961s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank1 [2026-10-28 12:31:13,937] INFO : Step 1600/50000 rank1 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2113 | Classification_binary: 0.4553 | Regression_cos_phi: 0.2667 | Regression_energy: 0.2531 | Regression_eta: 0.2143 | Regression_pt: 0.2627 | Regression_sin_phi: 0.3395 | LR: 9.98e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8068s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank1 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank1 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.3623 | Regression_cos_phi: 0.2024 | Regression_energy: 0.2248 | Regression_eta: 0.1365 | Regression_pt: 0.2245 | Regression_sin_phi: 0.1681 | LR: 9.97e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6756s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank1 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank1 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1756 | Classification_binary: 0.3735 | Regression_cos_phi: 0.1794 | Regression_energy: 0.2383 | Regression_eta: 0.1495 | Regression_pt: 0.2444 | Regression_sin_phi: 0.2178 | LR: 9.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6766s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank1 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank1 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1622 | Classification_binary: 0.3954 | Regression_cos_phi: 0.1605 | Regression_energy: 0.2131 | Regression_eta: 0.1439 | Regression_pt: 0.2218 | Regression_sin_phi: 0.1603 | LR: 9.97e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.4276s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank1 [2026-10-28 12:50:22,594] INFO : Step 2000/50000 rank1 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1873 | Classification_binary: 0.3794 | Regression_cos_phi: 0.1969 | Regression_energy: 0.2351 | Regression_eta: 0.2059 | Regression_pt: 0.2392 | Regression_sin_phi: 0.2196 | LR: 9.96e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9895s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank1 [2026-10-28 12:50:23,688] INFO : Running validation on rank1 +rank1 [2026-10-28 12:50:23,688] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 12:50:23,688] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 12:51:19,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:51:20,336] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 12:51:47,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:51:48,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:51:49,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:51:51,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:52:01,118] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 12:52:01,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:52:02,241] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 12:52:18,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:52:19,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:52:20,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:52:21,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:52:31,142] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:52:31,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:52:32,092] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 12:52:48,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:52:49,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:52:50,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:52:51,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:53:01,030] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank1 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1242 | Classification_binary: 0.3102 | Regression_cos_phi: 0.1772 | Regression_energy: 0.2428 | Regression_eta: 0.1561 | Regression_pt: 0.2477 | Regression_sin_phi: 0.1900 | LR: 9.96e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6522s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank1 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank1 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1511 | Classification_binary: 0.3913 | Regression_cos_phi: 0.2228 | Regression_energy: 0.2324 | Regression_eta: 0.1664 | Regression_pt: 0.2417 | Regression_sin_phi: 0.1926 | LR: 9.96e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.4343s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank1 [2026-10-28 13:08:12,324] INFO : Step 2300/50000 rank1 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 0.4220 | Regression_cos_phi: 0.2045 | Regression_energy: 0.2058 | Regression_eta: 0.2143 | Regression_pt: 0.2166 | Regression_sin_phi: 0.2327 | LR: 9.95e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.1088s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank1 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank1 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1404 | Classification_binary: 0.3483 | Regression_cos_phi: 0.1681 | Regression_energy: 0.2230 | Regression_eta: 0.1469 | Regression_pt: 0.2266 | Regression_sin_phi: 0.1967 | LR: 9.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2185s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank1 [2026-10-28 13:18:06,914] INFO : Step 2500/50000 rank1 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1832 | Classification_binary: 0.4396 | Regression_cos_phi: 0.2456 | Regression_energy: 0.2065 | Regression_eta: 0.1671 | Regression_pt: 0.2101 | Regression_sin_phi: 0.2250 | LR: 9.94e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.0387s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank1 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank1 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2003 | Classification_binary: 0.4255 | Regression_cos_phi: 0.2769 | Regression_energy: 0.2307 | Regression_eta: 0.1454 | Regression_pt: 0.2403 | Regression_sin_phi: 0.2105 | LR: 9.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3775s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank1 [2026-10-28 13:27:39,805] INFO : Step 2700/50000 rank1 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1177 | Classification_binary: 0.3311 | Regression_cos_phi: 0.1272 | Regression_energy: 0.2155 | Regression_eta: 0.1276 | Regression_pt: 0.2270 | Regression_sin_phi: 0.1764 | LR: 9.94e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5205s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank1 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank1 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1248 | Classification_binary: 0.2619 | Regression_cos_phi: 0.1506 | Regression_energy: 0.2271 | Regression_eta: 0.1042 | Regression_pt: 0.2346 | Regression_sin_phi: 0.1441 | LR: 9.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2916s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank1 [2026-10-28 13:37:09,430] INFO : Step 2900/50000 rank1 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1698 | Classification_binary: 0.4187 | Regression_cos_phi: 0.2201 | Regression_energy: 0.2323 | Regression_eta: 0.1987 | Regression_pt: 0.2430 | Regression_sin_phi: 0.2290 | LR: 9.93e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5097s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank1 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank1 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1426 | Classification_binary: 0.3673 | Regression_cos_phi: 0.2779 | Regression_energy: 0.2116 | Regression_eta: 0.1222 | Regression_pt: 0.2220 | Regression_sin_phi: 0.1800 | LR: 9.92e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5901s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank1 [2026-10-28 13:42:04,672] INFO : Running validation on rank1 +rank1 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:42:57,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:42:58,471] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 13:43:25,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:43:26,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:43:27,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:43:28,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:43:38,233] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:43:39,343] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 13:43:55,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:43:56,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:43:57,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:43:58,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:44:08,253] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:44:09,100] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 13:44:24,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:44:25,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:44:26,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:44:26,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:44:36,942] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 13:49:29,793] INFO : Step 3100/50000 rank1 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1121 | Classification_binary: 0.2658 | Regression_cos_phi: 0.1618 | Regression_energy: 0.1938 | Regression_eta: 0.0925 | Regression_pt: 0.1962 | Regression_sin_phi: 0.1277 | LR: 9.91e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.8430s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank1 [2026-10-28 13:54:22,656] INFO : Step 3200/50000 rank1 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1571 | Classification_binary: 0.3986 | Regression_cos_phi: 0.2633 | Regression_energy: 0.1982 | Regression_eta: 0.1843 | Regression_pt: 0.2029 | Regression_sin_phi: 0.2304 | LR: 9.91e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0530s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank1 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank1 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1756 | Classification_binary: 0.3584 | Regression_cos_phi: 0.1599 | Regression_energy: 0.1934 | Regression_eta: 0.1305 | Regression_pt: 0.2005 | Regression_sin_phi: 0.1854 | LR: 9.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7411s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank1 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank1 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1525 | Classification_binary: 0.3245 | Regression_cos_phi: 0.1769 | Regression_energy: 0.2167 | Regression_eta: 0.1517 | Regression_pt: 0.2238 | Regression_sin_phi: 0.1383 | LR: 9.90e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.9443s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank1 [2026-10-28 14:08:16,395] INFO : Step 3500/50000 rank1 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1727 | Classification_binary: 0.3459 | Regression_cos_phi: 0.2271 | Regression_energy: 0.2020 | Regression_eta: 0.1616 | Regression_pt: 0.2125 | Regression_sin_phi: 0.1804 | LR: 9.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7453s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank1 [2026-10-28 14:13:02,150] INFO : Step 3600/50000 rank1 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1881 | Classification_binary: 0.4486 | Regression_cos_phi: 0.2746 | Regression_energy: 0.1997 | Regression_eta: 0.1981 | Regression_pt: 0.2071 | Regression_sin_phi: 0.2651 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8774s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank1 [2026-10-28 14:17:42,402] INFO : Step 3700/50000 rank1 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1320 | Classification_binary: 0.3099 | Regression_cos_phi: 0.1479 | Regression_energy: 0.1861 | Regression_eta: 0.1278 | Regression_pt: 0.1916 | Regression_sin_phi: 0.1359 | LR: 9.88e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.2564s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank1 [2026-10-28 14:22:44,342] INFO : Step 3800/50000 rank1 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.3154 | Regression_cos_phi: 0.2123 | Regression_energy: 0.1666 | Regression_eta: 0.1400 | Regression_pt: 0.1743 | Regression_sin_phi: 0.2042 | LR: 9.87e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1812s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank1 [2026-10-28 14:27:33,639] INFO : Step 3900/50000 rank1 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1646 | Classification_binary: 0.3757 | Regression_cos_phi: 0.2388 | Regression_energy: 0.2280 | Regression_eta: 0.1673 | Regression_pt: 0.2348 | Regression_sin_phi: 0.2158 | LR: 9.87e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8229s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank1 [2026-10-28 14:32:38,644] INFO : Step 4000/50000 rank1 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1355 | Classification_binary: 0.3615 | Regression_cos_phi: 0.2048 | Regression_energy: 0.1816 | Regression_eta: 0.1558 | Regression_pt: 0.1880 | Regression_sin_phi: 0.1924 | LR: 9.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5861s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank1 [2026-10-28 14:32:39,608] INFO : Running validation on rank1 +rank1 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 14:33:34,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:33:34,720] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 14:34:03,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 14:34:05,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 14:34:08,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 14:34:10,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 14:34:20,589] INFO : Time taken to make predictions on device 1 is: 0.33 min +rank1 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:34:24,281] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 14:34:40,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 14:34:42,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 14:34:44,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 14:34:45,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 14:34:55,112] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:34:56,346] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 14:35:11,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 14:35:12,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 14:35:13,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 14:35:14,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 14:35:24,307] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank1 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1394 | Classification_binary: 0.2739 | Regression_cos_phi: 0.1668 | Regression_energy: 0.2123 | Regression_eta: 0.1645 | Regression_pt: 0.2186 | Regression_sin_phi: 0.2062 | LR: 9.85e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.8504s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank1 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank1 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2032 | Classification_binary: 0.3741 | Regression_cos_phi: 0.2528 | Regression_energy: 0.1984 | Regression_eta: 0.1982 | Regression_pt: 0.2091 | Regression_sin_phi: 0.2033 | LR: 9.84e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5080s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank1 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank1 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1898 | Classification_binary: 0.4278 | Regression_cos_phi: 0.2432 | Regression_energy: 0.1958 | Regression_eta: 0.1944 | Regression_pt: 0.2010 | Regression_sin_phi: 0.2600 | LR: 9.84e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4604s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank1 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank1 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1272 | Classification_binary: 0.2830 | Regression_cos_phi: 0.1918 | Regression_energy: 0.1799 | Regression_eta: 0.1145 | Regression_pt: 0.1854 | Regression_sin_phi: 0.1703 | LR: 9.83e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.1732s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank1 [2026-10-28 14:59:28,686] INFO : Step 4500/50000 rank1 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1186 | Classification_binary: 0.2697 | Regression_cos_phi: 0.2411 | Regression_energy: 0.1772 | Regression_eta: 0.1205 | Regression_pt: 0.1833 | Regression_sin_phi: 0.1527 | LR: 9.82e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2500s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank1 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank1 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1404 | Classification_binary: 0.4580 | Regression_cos_phi: 0.2026 | Regression_energy: 0.1825 | Regression_eta: 0.1622 | Regression_pt: 0.1877 | Regression_sin_phi: 0.1737 | LR: 9.81e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7023s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank1 [2026-10-28 15:09:18,560] INFO : Step 4700/50000 rank1 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1117 | Classification_binary: 0.3279 | Regression_cos_phi: 0.1411 | Regression_energy: 0.1728 | Regression_eta: 0.1093 | Regression_pt: 0.1796 | Regression_sin_phi: 0.1631 | LR: 9.81e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7051s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank1 [2026-10-28 15:14:30,753] INFO : Step 4800/50000 rank1 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1200 | Classification_binary: 0.2914 | Regression_cos_phi: 0.1750 | Regression_energy: 0.1635 | Regression_eta: 0.1258 | Regression_pt: 0.1693 | Regression_sin_phi: 0.1388 | LR: 9.80e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4400s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank1 [2026-10-28 15:19:27,570] INFO : Step 4900/50000 rank1 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1625 | Classification_binary: 0.3458 | Regression_cos_phi: 0.2337 | Regression_energy: 0.1959 | Regression_eta: 0.1644 | Regression_pt: 0.2034 | Regression_sin_phi: 0.2270 | LR: 9.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5354s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank1 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank1 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1522 | Classification_binary: 0.3189 | Regression_cos_phi: 0.2255 | Regression_energy: 0.2052 | Regression_eta: 0.1564 | Regression_pt: 0.2088 | Regression_sin_phi: 0.2062 | LR: 9.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4829s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank1 [2026-10-28 15:24:24,982] INFO : Running validation on rank1 +rank1 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 15:25:19,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:25:19,571] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 15:25:44,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 15:25:45,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 15:25:46,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 15:25:48,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 15:25:58,013] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 15:25:58,743] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:25:59,250] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 15:26:15,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 15:26:16,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 15:26:17,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 15:26:18,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 15:26:28,430] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 15:26:28,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 15:26:45,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 15:26:46,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 15:26:46,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 15:26:47,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 15:26:57,726] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 15:32:01,929] INFO : Step 5100/50000 rank1 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1310 | Regression_energy: 0.1585 | Regression_eta: 0.1087 | Regression_pt: 0.1631 | Regression_sin_phi: 0.1001 | LR: 9.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.6408s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank1 [2026-10-28 15:36:53,510] INFO : Step 5200/50000 rank1 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1431 | Classification_binary: 0.3149 | Regression_cos_phi: 0.1800 | Regression_energy: 0.1635 | Regression_eta: 0.1331 | Regression_pt: 0.1731 | Regression_sin_phi: 0.1730 | LR: 9.76e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.6357s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank1 [2026-10-28 15:41:36,524] INFO : Step 5300/50000 rank1 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1579 | Classification_binary: 0.3811 | Regression_cos_phi: 0.2253 | Regression_energy: 0.1990 | Regression_eta: 0.1302 | Regression_pt: 0.2067 | Regression_sin_phi: 0.1688 | LR: 9.75e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9229s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank1 [2026-10-28 15:46:33,435] INFO : Step 5400/50000 rank1 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1263 | Classification_binary: 0.3181 | Regression_cos_phi: 0.1952 | Regression_energy: 0.1559 | Regression_eta: 0.1610 | Regression_pt: 0.1613 | Regression_sin_phi: 0.1922 | LR: 9.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4635s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank1 [2026-10-28 15:51:26,878] INFO : Step 5500/50000 rank1 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1412 | Classification_binary: 0.3676 | Regression_cos_phi: 0.2337 | Regression_energy: 0.2065 | Regression_eta: 0.1537 | Regression_pt: 0.2136 | Regression_sin_phi: 0.1927 | LR: 9.73e-04 | DataLoad Time: 0.0185s | Model Forward Time: 6.5393s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank1 [2026-10-28 15:56:25,711] INFO : Step 5600/50000 rank1 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1601 | Classification_binary: 0.3946 | Regression_cos_phi: 0.1819 | Regression_energy: 0.1630 | Regression_eta: 0.1322 | Regression_pt: 0.1689 | Regression_sin_phi: 0.2058 | LR: 9.72e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.3195s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank1 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank1 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.3039 | Regression_cos_phi: 0.1271 | Regression_energy: 0.1747 | Regression_eta: 0.1195 | Regression_pt: 0.1819 | Regression_sin_phi: 0.1215 | LR: 9.71e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5510s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank1 [2026-10-28 16:06:09,458] INFO : Step 5800/50000 rank1 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.2369 | Regression_cos_phi: 0.1501 | Regression_energy: 0.1519 | Regression_eta: 0.1192 | Regression_pt: 0.1584 | Regression_sin_phi: 0.1280 | LR: 9.70e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2443s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank1 [2026-10-28 16:11:19,901] INFO : Step 5900/50000 rank1 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1479 | Classification_binary: 0.3191 | Regression_cos_phi: 0.1704 | Regression_energy: 0.1765 | Regression_eta: 0.1281 | Regression_pt: 0.1850 | Regression_sin_phi: 0.1733 | LR: 9.69e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.1474s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank1 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank1 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1223 | Classification_binary: 0.3420 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1555 | Regression_eta: 0.1311 | Regression_pt: 0.1592 | Regression_sin_phi: 0.1521 | LR: 9.68e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.4951s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank1 [2026-10-28 16:16:41,623] INFO : Running validation on rank1 +rank1 [2026-10-28 16:16:41,623] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:16:41,623] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:17:32,564] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:17:33,087] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 16:17:59,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:18:00,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:18:01,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:18:02,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:18:12,685] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:18:13,875] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 16:18:30,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:18:31,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:18:32,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:18:33,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:18:43,185] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:18:43,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:18:43,953] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 16:18:59,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:19:00,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:19:02,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:19:03,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:19:13,541] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank1 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1232 | Classification_binary: 0.2498 | Regression_cos_phi: 0.1525 | Regression_energy: 0.1563 | Regression_eta: 0.1136 | Regression_pt: 0.1573 | Regression_sin_phi: 0.1891 | LR: 9.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9554s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank1 [2026-10-28 16:29:26,898] INFO : Step 6200/50000 rank1 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1305 | Classification_binary: 0.2649 | Regression_cos_phi: 0.1621 | Regression_energy: 0.1468 | Regression_eta: 0.1292 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1669 | LR: 9.66e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0891s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank1 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank1 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1495 | Classification_binary: 0.3576 | Regression_cos_phi: 0.1568 | Regression_energy: 0.1681 | Regression_eta: 0.1243 | Regression_pt: 0.1763 | Regression_sin_phi: 0.2263 | LR: 9.65e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0292s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank1 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank1 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.3010 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1725 | Regression_eta: 0.1164 | Regression_pt: 0.1766 | Regression_sin_phi: 0.1598 | LR: 9.64e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7765s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank1 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank1 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1267 | Classification_binary: 0.2443 | Regression_cos_phi: 0.1428 | Regression_energy: 0.1508 | Regression_eta: 0.0933 | Regression_pt: 0.1575 | Regression_sin_phi: 0.1697 | LR: 9.63e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4995s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank1 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank1 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1397 | Classification_binary: 0.3315 | Regression_cos_phi: 0.1504 | Regression_energy: 0.1798 | Regression_eta: 0.1215 | Regression_pt: 0.1862 | Regression_sin_phi: 0.2076 | LR: 9.62e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8077s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank1 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank1 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1418 | Classification_binary: 0.3304 | Regression_cos_phi: 0.1347 | Regression_energy: 0.1609 | Regression_eta: 0.1313 | Regression_pt: 0.1668 | Regression_sin_phi: 0.1918 | LR: 9.61e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.3358s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank1 [2026-10-28 16:58:34,481] INFO : Step 6800/50000 rank1 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1096 | Classification_binary: 0.2740 | Regression_cos_phi: 0.1313 | Regression_energy: 0.1441 | Regression_eta: 0.0921 | Regression_pt: 0.1465 | Regression_sin_phi: 0.1505 | LR: 9.60e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.4623s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank1 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank1 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1045 | Classification_binary: 0.3293 | Regression_cos_phi: 0.1497 | Regression_energy: 0.1469 | Regression_eta: 0.1198 | Regression_pt: 0.1521 | Regression_sin_phi: 0.1598 | LR: 9.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6879s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank1 [2026-10-28 17:08:29,914] INFO : Step 7000/50000 rank1 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3171 | Regression_cos_phi: 0.1406 | Regression_energy: 0.1761 | Regression_eta: 0.1442 | Regression_pt: 0.1818 | Regression_sin_phi: 0.1820 | LR: 9.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9214s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank1 [2026-10-28 17:08:30,791] INFO : Running validation on rank1 +rank1 [2026-10-28 17:08:30,791] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 17:08:30,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 17:09:23,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:09:24,266] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 17:09:51,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 17:09:53,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 17:09:55,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 17:09:56,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 17:10:06,697] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-28 17:10:07,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:10:08,263] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 17:10:24,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 17:10:25,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 17:10:26,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 17:10:27,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 17:10:37,255] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:10:37,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:10:38,448] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 17:10:53,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 17:10:54,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 17:10:55,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 17:10:56,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 17:11:06,165] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 17:16:27,595] INFO : Step 7100/50000 rank1 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1180 | Classification_binary: 0.2666 | Regression_cos_phi: 0.1052 | Regression_energy: 0.1442 | Regression_eta: 0.0804 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1076 | LR: 9.56e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1469s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank1 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank1 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1109 | Classification_binary: 0.3007 | Regression_cos_phi: 0.1645 | Regression_energy: 0.1433 | Regression_eta: 0.0944 | Regression_pt: 0.1485 | Regression_sin_phi: 0.1613 | LR: 9.55e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3449s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank1 [2026-10-28 17:26:34,942] INFO : Step 7300/50000 rank1 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1201 | Classification_binary: 0.3434 | Regression_cos_phi: 0.1811 | Regression_energy: 0.1470 | Regression_eta: 0.1172 | Regression_pt: 0.1515 | Regression_sin_phi: 0.1428 | LR: 9.53e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4021s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank1 [2026-10-28 17:31:34,683] INFO : Step 7400/50000 rank1 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0971 | Classification_binary: 0.3031 | Regression_cos_phi: 0.1881 | Regression_energy: 0.1454 | Regression_eta: 0.1068 | Regression_pt: 0.1480 | Regression_sin_phi: 0.1628 | LR: 9.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5936s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank1 [2026-10-28 17:36:30,884] INFO : Step 7500/50000 rank1 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0893 | Classification_binary: 0.2370 | Regression_cos_phi: 0.1357 | Regression_energy: 0.1519 | Regression_eta: 0.0953 | Regression_pt: 0.1581 | Regression_sin_phi: 0.1444 | LR: 9.51e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6591s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank1 [2026-10-28 17:41:09,112] INFO : Step 7600/50000 rank1 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1594 | Classification_binary: 0.3399 | Regression_cos_phi: 0.1979 | Regression_energy: 0.1543 | Regression_eta: 0.1246 | Regression_pt: 0.1605 | Regression_sin_phi: 0.1893 | LR: 9.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7892s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank1 [2026-10-28 17:46:03,243] INFO : Step 7700/50000 rank1 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1033 | Classification_binary: 0.3248 | Regression_cos_phi: 0.1503 | Regression_energy: 0.1441 | Regression_eta: 0.1137 | Regression_pt: 0.1512 | Regression_sin_phi: 0.1259 | LR: 9.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7820s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank1 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank1 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.2680 | Regression_cos_phi: 0.1795 | Regression_energy: 0.1670 | Regression_eta: 0.1147 | Regression_pt: 0.1726 | Regression_sin_phi: 0.1575 | LR: 9.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4017s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank1 [2026-10-28 17:56:05,209] INFO : Step 7900/50000 rank1 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1254 | Classification_binary: 0.3725 | Regression_cos_phi: 0.2073 | Regression_energy: 0.1525 | Regression_eta: 0.1382 | Regression_pt: 0.1592 | Regression_sin_phi: 0.1859 | LR: 9.46e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.3970s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank1 [2026-10-28 18:00:55,276] INFO : Step 8000/50000 rank1 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.3105 | Regression_cos_phi: 0.1755 | Regression_energy: 0.1381 | Regression_eta: 0.1181 | Regression_pt: 0.1416 | Regression_sin_phi: 0.1420 | LR: 9.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4817s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank1 [2026-10-28 18:00:56,227] INFO : Running validation on rank1 +rank1 [2026-10-28 18:00:56,227] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:00:56,227] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:01:47,759] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:01:48,314] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 18:02:14,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:02:15,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:02:16,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:02:17,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:02:27,939] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:02:28,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 18:02:44,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:02:45,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:02:46,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:02:47,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:02:57,464] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:02:58,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:02:59,068] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 18:03:14,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:03:15,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:03:16,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:03:17,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:03:27,346] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 18:08:30,022] INFO : Step 8100/50000 rank1 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1099 | Classification_binary: 0.2903 | Regression_cos_phi: 0.1467 | Regression_energy: 0.1512 | Regression_eta: 0.1165 | Regression_pt: 0.1560 | Regression_sin_phi: 0.1242 | LR: 9.43e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.3412s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank1 [2026-10-28 18:13:41,921] INFO : Step 8200/50000 rank1 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1296 | Classification_binary: 0.2650 | Regression_cos_phi: 0.1403 | Regression_energy: 0.1455 | Regression_eta: 0.0838 | Regression_pt: 0.1500 | Regression_sin_phi: 0.1539 | LR: 9.42e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3245s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank1 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank1 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3488 | Regression_cos_phi: 0.1970 | Regression_energy: 0.1483 | Regression_eta: 0.1395 | Regression_pt: 0.1554 | Regression_sin_phi: 0.2193 | LR: 9.40e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6359s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank1 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank1 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1323 | Classification_binary: 0.3565 | Regression_cos_phi: 0.1838 | Regression_energy: 0.1379 | Regression_eta: 0.1457 | Regression_pt: 0.1446 | Regression_sin_phi: 0.1891 | LR: 9.39e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank1 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank1 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0950 | Regression_energy: 0.1216 | Regression_eta: 0.0632 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0585 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5042s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank1 [2026-10-28 18:34:09,471] INFO : Step 8600/50000 rank1 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1233 | Classification_binary: 0.3546 | Regression_cos_phi: 0.1964 | Regression_energy: 0.1513 | Regression_eta: 0.1356 | Regression_pt: 0.1598 | Regression_sin_phi: 0.1712 | LR: 9.36e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7731s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank1 [2026-10-28 18:39:14,578] INFO : Step 8700/50000 rank1 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.3281 | Regression_cos_phi: 0.1945 | Regression_energy: 0.1468 | Regression_eta: 0.1350 | Regression_pt: 0.1511 | Regression_sin_phi: 0.1782 | LR: 9.34e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.7184s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank1 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank1 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0973 | Classification_binary: 0.2348 | Regression_cos_phi: 0.1503 | Regression_energy: 0.1143 | Regression_eta: 0.0850 | Regression_pt: 0.1185 | Regression_sin_phi: 0.1140 | LR: 9.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.4108s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank1 [2026-10-28 18:49:28,516] INFO : Step 8900/50000 rank1 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1242 | Classification_binary: 0.2950 | Regression_cos_phi: 0.2036 | Regression_energy: 0.1410 | Regression_eta: 0.1218 | Regression_pt: 0.1447 | Regression_sin_phi: 0.2313 | LR: 9.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7912s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank1 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank1 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1570 | Regression_energy: 0.1340 | Regression_eta: 0.0888 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1428 | LR: 9.30e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8382s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank1 [2026-10-28 18:54:54,935] INFO : Running validation on rank1 +rank1 [2026-10-28 18:54:54,935] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:54:54,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:55:52,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:55:53,257] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 18:56:19,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:56:21,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:56:22,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:56:24,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:56:34,644] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-28 18:56:34,836] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:56:35,474] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 18:56:51,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:56:52,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:56:53,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:56:54,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:57:04,622] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:57:05,964] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 18:57:21,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:57:22,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:57:23,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:57:24,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:57:34,466] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 19:03:08,597] INFO : Step 9100/50000 rank1 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2531 | Regression_cos_phi: 0.1467 | Regression_energy: 0.1131 | Regression_eta: 0.0864 | Regression_pt: 0.1164 | Regression_sin_phi: 0.1390 | LR: 9.28e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.1690s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank1 [2026-10-28 19:08:18,223] INFO : Step 9200/50000 rank1 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0864 | Classification_binary: 0.2395 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1512 | Regression_eta: 0.0927 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1197 | LR: 9.27e-04 | DataLoad Time: 0.0245s | Model Forward Time: 6.5449s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank1 [2026-10-28 19:13:30,253] INFO : Step 9300/50000 rank1 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2987 | Regression_cos_phi: 0.1291 | Regression_energy: 0.1173 | Regression_eta: 0.1000 | Regression_pt: 0.1224 | Regression_sin_phi: 0.1223 | LR: 9.25e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3918s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank1 [2026-10-28 19:18:55,613] INFO : Step 9400/50000 rank1 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2504 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1385 | Regression_eta: 0.0832 | Regression_pt: 0.1423 | Regression_sin_phi: 0.1078 | LR: 9.24e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2032s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank1 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank1 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2514 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1265 | Regression_eta: 0.0713 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0947 | LR: 9.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2846s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank1 [2026-10-28 19:29:34,438] INFO : Step 9600/50000 rank1 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2387 | Regression_cos_phi: 0.1186 | Regression_energy: 0.1297 | Regression_eta: 0.0745 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0964 | LR: 9.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6039s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank1 [2026-10-28 19:34:53,790] INFO : Step 9700/50000 rank1 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.3007 | Regression_cos_phi: 0.1405 | Regression_energy: 0.1205 | Regression_eta: 0.0965 | Regression_pt: 0.1257 | Regression_sin_phi: 0.1320 | LR: 9.19e-04 | DataLoad Time: 0.0096s | Model Forward Time: 4.9902s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank1 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank1 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2418 | Regression_cos_phi: 0.1400 | Regression_energy: 0.1024 | Regression_eta: 0.0755 | Regression_pt: 0.1038 | Regression_sin_phi: 0.1040 | LR: 9.17e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5617s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank1 [2026-10-28 19:44:46,547] INFO : Step 9900/50000 rank1 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2589 | Regression_cos_phi: 0.1628 | Regression_energy: 0.1332 | Regression_eta: 0.0980 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1626 | LR: 9.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9293s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank1 [2026-10-28 19:49:37,631] INFO : Step 10000/50000 rank1 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.3114 | Regression_cos_phi: 0.1276 | Regression_energy: 0.1397 | Regression_eta: 0.1078 | Regression_pt: 0.1443 | Regression_sin_phi: 0.1540 | LR: 9.14e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.6251s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank1 [2026-10-28 19:49:38,689] INFO : Running validation on rank1 +rank1 [2026-10-28 19:49:38,690] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 19:49:38,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 19:50:35,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:50:35,626] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 19:51:01,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 19:51:02,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 19:51:03,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 19:51:04,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 19:51:14,970] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 19:51:15,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:51:16,750] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 19:51:32,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 19:51:33,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 19:51:34,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 19:51:35,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 19:51:45,846] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 19:51:46,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:51:46,687] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 19:52:02,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 19:52:03,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 19:52:03,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 19:52:04,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 19:52:14,781] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 19:57:27,967] INFO : Step 10100/50000 rank1 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1139 | Regression_eta: 0.0661 | Regression_pt: 0.1186 | Regression_sin_phi: 0.1129 | LR: 9.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7458s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank1 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank1 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2035 | Regression_cos_phi: 0.1212 | Regression_energy: 0.1599 | Regression_eta: 0.0828 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0952 | LR: 9.11e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6810s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank1 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank1 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.3032 | Regression_cos_phi: 0.1306 | Regression_energy: 0.1249 | Regression_eta: 0.0916 | Regression_pt: 0.1308 | Regression_sin_phi: 0.1320 | LR: 9.09e-04 | DataLoad Time: 0.0237s | Model Forward Time: 5.1986s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank1 [2026-10-28 20:12:49,147] INFO : Step 10400/50000 rank1 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1083 | Regression_energy: 0.1201 | Regression_eta: 0.0658 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1083 | LR: 9.07e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2213s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank1 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank1 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2152 | Regression_cos_phi: 0.1013 | Regression_energy: 0.1028 | Regression_eta: 0.0612 | Regression_pt: 0.1059 | Regression_sin_phi: 0.0990 | LR: 9.06e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3226s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank1 [2026-10-28 20:23:17,056] INFO : Step 10600/50000 rank1 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2200 | Regression_cos_phi: 0.1076 | Regression_energy: 0.1017 | Regression_eta: 0.0666 | Regression_pt: 0.1066 | Regression_sin_phi: 0.1055 | LR: 9.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0930s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank1 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank1 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0897 | Classification_binary: 0.2949 | Regression_cos_phi: 0.1306 | Regression_energy: 0.1130 | Regression_eta: 0.1164 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1171 | LR: 9.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8720s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank1 [2026-10-28 20:33:28,995] INFO : Step 10800/50000 rank1 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0836 | Regression_energy: 0.1048 | Regression_eta: 0.0635 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0843 | LR: 9.00e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.6819s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank1 [2026-10-28 20:38:44,061] INFO : Step 10900/50000 rank1 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0955 | Classification_binary: 0.2384 | Regression_cos_phi: 0.1511 | Regression_energy: 0.1013 | Regression_eta: 0.1107 | Regression_pt: 0.1082 | Regression_sin_phi: 0.1438 | LR: 8.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7067s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank1 [2026-10-28 20:43:46,527] INFO : Step 11000/50000 rank1 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2884 | Regression_cos_phi: 0.1178 | Regression_energy: 0.1172 | Regression_eta: 0.0808 | Regression_pt: 0.1227 | Regression_sin_phi: 0.1084 | LR: 8.97e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4450s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank1 [2026-10-28 20:43:47,415] INFO : Running validation on rank1 +rank1 [2026-10-28 20:43:47,415] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 20:43:47,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 20:44:41,298] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 20:45:09,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 20:45:10,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 20:45:11,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 20:45:12,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 20:45:22,997] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 20:45:23,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:45:24,126] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 20:45:40,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 20:45:41,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 20:45:42,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 20:45:43,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 20:45:53,652] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 20:46:11,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 20:46:12,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 20:46:13,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 20:46:14,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 20:46:24,836] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 20:51:55,950] INFO : Step 11100/50000 rank1 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0922 | Regression_energy: 0.1021 | Regression_eta: 0.0585 | Regression_pt: 0.1065 | Regression_sin_phi: 0.1143 | LR: 8.95e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8328s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank1 [2026-10-28 20:57:10,090] INFO : Step 11200/50000 rank1 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0997 | Regression_eta: 0.0559 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0900 | LR: 8.93e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8512s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank1 [2026-10-28 21:02:18,460] INFO : Step 11300/50000 rank1 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1098 | Classification_binary: 0.2755 | Regression_cos_phi: 0.1409 | Regression_energy: 0.0984 | Regression_eta: 0.0964 | Regression_pt: 0.1001 | Regression_sin_phi: 0.1495 | LR: 8.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank1 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank1 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2972 | Regression_cos_phi: 0.1053 | Regression_energy: 0.1031 | Regression_eta: 0.0708 | Regression_pt: 0.1057 | Regression_sin_phi: 0.1084 | LR: 8.89e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9853s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank1 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank1 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2182 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0895 | Regression_eta: 0.0571 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0930 | LR: 8.88e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3391s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank1 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank1 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2497 | Regression_cos_phi: 0.1376 | Regression_energy: 0.0957 | Regression_eta: 0.0663 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0904 | LR: 8.86e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank1 [2026-10-28 21:22:13,463] INFO : Step 11700/50000 rank1 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.3154 | Regression_cos_phi: 0.1271 | Regression_energy: 0.0948 | Regression_eta: 0.0724 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1347 | LR: 8.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank1 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank1 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2066 | Regression_cos_phi: 0.1473 | Regression_energy: 0.0868 | Regression_eta: 0.1018 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0925 | LR: 8.82e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5587s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank1 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank1 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2461 | Regression_cos_phi: 0.1477 | Regression_energy: 0.0931 | Regression_eta: 0.0694 | Regression_pt: 0.0950 | Regression_sin_phi: 0.1053 | LR: 8.80e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9852s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank1 [2026-10-28 21:36:37,258] INFO : Step 12000/50000 rank1 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0843 | Regression_energy: 0.1028 | Regression_eta: 0.0556 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0766 | LR: 8.78e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0197s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank1 [2026-10-28 21:36:38,362] INFO : Running validation on rank1 +rank1 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:37:31,336] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 21:38:00,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:38:01,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:38:02,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:38:03,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:38:13,545] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:38:14,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:38:14,612] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 21:38:30,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:38:32,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:38:33,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:38:34,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:38:44,961] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 21:38:45,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:38:46,120] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 21:39:01,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:39:02,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:39:03,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:39:04,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:39:14,092] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank1 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.2392 | Regression_cos_phi: 0.1273 | Regression_energy: 0.1002 | Regression_eta: 0.0517 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0962 | LR: 8.76e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1504s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank1 [2026-10-28 21:49:00,968] INFO : Step 12200/50000 rank1 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0822 | Regression_energy: 0.1357 | Regression_eta: 0.0631 | Regression_pt: 0.1386 | Regression_sin_phi: 0.1278 | LR: 8.74e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5154s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank1 [2026-10-28 21:53:55,878] INFO : Step 12300/50000 rank1 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0951 | Regression_eta: 0.0678 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1374 | LR: 8.72e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6861s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank1 [2026-10-28 21:58:40,948] INFO : Step 12400/50000 rank1 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.2892 | Regression_cos_phi: 0.1269 | Regression_energy: 0.0876 | Regression_eta: 0.0647 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0967 | LR: 8.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2699s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank1 [2026-10-28 22:03:23,385] INFO : Step 12500/50000 rank1 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0885 | Regression_eta: 0.0722 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0812 | LR: 8.68e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6710s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank1 [2026-10-28 22:08:04,829] INFO : Step 12600/50000 rank1 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2302 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0923 | Regression_eta: 0.0794 | Regression_pt: 0.0960 | Regression_sin_phi: 0.1073 | LR: 8.66e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.1212s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank1 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank1 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2546 | Regression_cos_phi: 0.1206 | Regression_energy: 0.1078 | Regression_eta: 0.0530 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0910 | LR: 8.64e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3610s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank1 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank1 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2750 | Regression_cos_phi: 0.1228 | Regression_energy: 0.0893 | Regression_eta: 0.0929 | Regression_pt: 0.0944 | Regression_sin_phi: 0.1267 | LR: 8.62e-04 | DataLoad Time: 0.0239s | Model Forward Time: 6.0957s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank1 [2026-10-28 22:22:32,702] INFO : Step 12900/50000 rank1 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.1999 | Regression_cos_phi: 0.1255 | Regression_energy: 0.0850 | Regression_eta: 0.0896 | Regression_pt: 0.0863 | Regression_sin_phi: 0.1232 | LR: 8.60e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6618s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank1 [2026-10-28 22:27:21,815] INFO : Step 13000/50000 rank1 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0908 | Regression_eta: 0.0493 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0957 | LR: 8.58e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9249s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank1 [2026-10-28 22:27:23,204] INFO : Running validation on rank1 +rank1 [2026-10-28 22:27:23,204] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 22:27:23,204] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 22:28:27,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:28:28,169] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 22:28:54,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 22:28:55,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 22:28:56,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 22:28:57,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 22:29:07,223] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:29:08,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:29:08,606] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 22:29:24,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 22:29:25,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 22:29:26,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 22:29:27,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 22:29:37,592] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:29:38,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:29:38,494] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 22:29:54,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 22:29:55,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 22:29:55,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 22:29:56,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 22:30:06,821] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 22:35:15,707] INFO : Step 13100/50000 rank1 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2205 | Regression_cos_phi: 0.1371 | Regression_energy: 0.0814 | Regression_eta: 0.0538 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0795 | LR: 8.56e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.4849s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank1 [2026-10-28 22:40:04,073] INFO : Step 13200/50000 rank1 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0857 | Regression_eta: 0.0440 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0716 | LR: 8.54e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.6849s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank1 [2026-10-28 22:44:58,538] INFO : Step 13300/50000 rank1 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2338 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0891 | Regression_eta: 0.0530 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0965 | LR: 8.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6736s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank1 [2026-10-28 22:49:50,192] INFO : Step 13400/50000 rank1 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1233 | Classification_binary: 0.3413 | Regression_cos_phi: 0.1178 | Regression_energy: 0.1209 | Regression_eta: 0.1052 | Regression_pt: 0.1254 | Regression_sin_phi: 0.1231 | LR: 8.50e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.6028s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank1 [2026-10-28 22:54:39,509] INFO : Step 13500/50000 rank1 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0859 | Regression_eta: 0.0393 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0601 | LR: 8.48e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4627s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank1 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank1 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1951 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0898 | Regression_eta: 0.0491 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0916 | LR: 8.45e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8500s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank1 [2026-10-28 23:04:05,987] INFO : Step 13700/50000 rank1 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2917 | Regression_cos_phi: 0.1056 | Regression_energy: 0.0982 | Regression_eta: 0.0713 | Regression_pt: 0.1023 | Regression_sin_phi: 0.1054 | LR: 8.43e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7306s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank1 [2026-10-28 23:09:16,293] INFO : Step 13800/50000 rank1 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0860 | Regression_eta: 0.0483 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0713 | LR: 8.41e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4783s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank1 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank1 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2382 | Regression_cos_phi: 0.1101 | Regression_energy: 0.1039 | Regression_eta: 0.0749 | Regression_pt: 0.1081 | Regression_sin_phi: 0.1239 | LR: 8.39e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2890s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank1 [2026-10-28 23:18:59,721] INFO : Step 14000/50000 rank1 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2192 | Regression_cos_phi: 0.1059 | Regression_energy: 0.1091 | Regression_eta: 0.0684 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0984 | LR: 8.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1753s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank1 [2026-10-28 23:19:00,859] INFO : Running validation on rank1 +rank1 [2026-10-28 23:19:00,859] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:19:53,517] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-28 23:20:19,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 23:20:20,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 23:20:21,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 23:20:22,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 23:20:32,425] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 23:20:33,202] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:20:33,744] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-28 23:20:49,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 23:20:50,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 23:20:51,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 23:20:52,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 23:21:02,466] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 23:21:03,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:21:04,039] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-28 23:21:19,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 23:21:19,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 23:21:20,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 23:21:21,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 23:21:31,589] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 23:26:43,451] INFO : Step 14100/50000 rank1 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2425 | Regression_cos_phi: 0.1001 | Regression_energy: 0.0876 | Regression_eta: 0.0808 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0823 | LR: 8.35e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.5356s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank1 [2026-10-28 23:31:31,720] INFO : Step 14200/50000 rank1 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0846 | Regression_eta: 0.0394 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0798 | LR: 8.32e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.6152s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank1 [2026-10-28 23:36:16,486] INFO : Step 14300/50000 rank1 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0812 | Regression_eta: 0.0521 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0789 | LR: 8.30e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6731s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank1 [2026-10-28 23:40:58,337] INFO : Step 14400/50000 rank1 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2837 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0860 | Regression_eta: 0.0648 | Regression_pt: 0.0889 | Regression_sin_phi: 0.1122 | LR: 8.28e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7763s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank1 [2026-10-28 23:45:42,940] INFO : Step 14500/50000 rank1 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0700 | Regression_eta: 0.0453 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0731 | LR: 8.26e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.6335s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank1 [2026-10-28 23:50:27,493] INFO : Step 14600/50000 rank1 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0889 | Regression_eta: 0.0700 | Regression_pt: 0.0909 | Regression_sin_phi: 0.1120 | LR: 8.24e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4473s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank1 [2026-10-28 23:55:12,843] INFO : Step 14700/50000 rank1 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2721 | Regression_cos_phi: 0.1175 | Regression_energy: 0.0932 | Regression_eta: 0.0565 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0718 | LR: 8.21e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6474s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank1 [2026-10-28 23:59:56,485] INFO : Step 14800/50000 rank1 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0939 | Regression_energy: 0.1099 | Regression_eta: 0.0691 | Regression_pt: 0.1135 | Regression_sin_phi: 0.1066 | LR: 8.19e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.7610s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank1 [2026-10-29 00:04:46,438] INFO : Step 14900/50000 rank1 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0671 | Regression_eta: 0.0649 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0619 | LR: 8.17e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1955s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank1 [2026-10-29 00:09:43,123] INFO : Step 15000/50000 rank1 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2237 | Regression_cos_phi: 0.1102 | Regression_energy: 0.0906 | Regression_eta: 0.0786 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0911 | LR: 8.15e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7477s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank1 [2026-10-29 00:09:44,170] INFO : Running validation on rank1 +rank1 [2026-10-29 00:09:44,170] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 00:09:44,170] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:10:38,762] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:10:39,314] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 00:11:06,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:11:07,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:11:08,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:11:09,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:11:19,586] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 00:11:20,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:11:20,773] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 00:11:36,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:11:37,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:11:38,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:11:39,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:11:49,638] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:11:50,985] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 00:12:06,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:12:07,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:12:08,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:12:09,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:12:19,237] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 00:17:30,254] INFO : Step 15100/50000 rank1 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2107 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1006 | Regression_eta: 0.0478 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0675 | LR: 8.12e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3988s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank1 [2026-10-29 00:22:16,929] INFO : Step 15200/50000 rank1 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0775 | Regression_eta: 0.0571 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0842 | LR: 8.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2289s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank1 [2026-10-29 00:27:05,719] INFO : Step 15300/50000 rank1 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0808 | Regression_eta: 0.0532 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0861 | LR: 8.08e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3164s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank1 [2026-10-29 00:31:57,726] INFO : Step 15400/50000 rank1 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1376 | Regression_energy: 0.0847 | Regression_eta: 0.0642 | Regression_pt: 0.0888 | Regression_sin_phi: 0.1288 | LR: 8.05e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6654s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank1 [2026-10-29 00:36:52,791] INFO : Step 15500/50000 rank1 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0818 | Regression_eta: 0.0450 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0762 | LR: 8.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank1 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank1 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0756 | Regression_eta: 0.0565 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0753 | LR: 8.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9289s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank1 [2026-10-29 00:46:43,874] INFO : Step 15700/50000 rank1 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.2297 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0859 | Regression_eta: 0.0426 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0745 | LR: 7.98e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7577s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank1 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank1 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0896 | Regression_eta: 0.0556 | Regression_pt: 0.0939 | Regression_sin_phi: 0.1000 | LR: 7.96e-04 | DataLoad Time: 0.0229s | Model Forward Time: 5.0254s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank1 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank1 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0834 | Regression_eta: 0.0388 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0667 | LR: 7.94e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank1 [2026-10-29 01:01:16,830] INFO : Step 16000/50000 rank1 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.2082 | Regression_cos_phi: 0.1110 | Regression_energy: 0.0867 | Regression_eta: 0.0599 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0759 | LR: 7.91e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.3025s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank1 [2026-10-29 01:01:17,882] INFO : Running validation on rank1 +rank1 [2026-10-29 01:01:17,882] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 01:01:17,882] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 01:02:10,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:02:11,392] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 01:02:39,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 01:02:41,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 01:02:42,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 01:02:44,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 01:02:54,516] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-29 01:02:55,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:02:55,637] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 01:03:11,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 01:03:12,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 01:03:13,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 01:03:14,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 01:03:24,994] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 01:03:25,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:03:26,085] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 01:03:41,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 01:03:42,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 01:03:43,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 01:03:44,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 01:03:54,442] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank1 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2096 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0565 | Regression_eta: 0.1047 | Regression_pt: 0.0582 | Regression_sin_phi: 0.1000 | LR: 7.89e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0665s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank1 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank1 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0776 | Regression_eta: 0.0588 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0773 | LR: 7.86e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2105s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank1 [2026-10-29 01:19:03,891] INFO : Step 16300/50000 rank1 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0984 | Regression_energy: 0.0899 | Regression_eta: 0.0762 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0859 | LR: 7.84e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.2111s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank1 [2026-10-29 01:23:56,182] INFO : Step 16400/50000 rank1 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0618 | Regression_eta: 0.0489 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0937 | LR: 7.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6819s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank1 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank1 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0784 | Regression_eta: 0.0679 | Regression_pt: 0.0797 | Regression_sin_phi: 0.1017 | LR: 7.79e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3077s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank1 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank1 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0853 | Regression_eta: 0.0733 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0788 | LR: 7.77e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1715s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank1 [2026-10-29 01:38:38,619] INFO : Step 16700/50000 rank1 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2288 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0838 | Regression_eta: 0.0558 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0810 | LR: 7.74e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.9119s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank1 [2026-10-29 01:43:30,909] INFO : Step 16800/50000 rank1 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0825 | Regression_eta: 0.0928 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0968 | LR: 7.72e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.5995s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank1 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank1 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0696 | Regression_eta: 0.0629 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0509 | LR: 7.69e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3813s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank1 [2026-10-29 01:53:19,532] INFO : Step 17000/50000 rank1 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2067 | Regression_cos_phi: 0.1122 | Regression_energy: 0.0793 | Regression_eta: 0.0689 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0833 | LR: 7.67e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.6773s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank1 [2026-10-29 01:53:20,625] INFO : Running validation on rank1 +rank1 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 01:54:26,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:54:27,390] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 01:55:04,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 01:55:05,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 01:55:06,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 01:55:07,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 01:55:17,864] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 01:55:18,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:55:18,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 01:55:34,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 01:55:35,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 01:55:37,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 01:55:38,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 01:55:48,481] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 01:55:49,891] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:55:50,413] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 01:56:06,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 01:56:06,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 01:56:07,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 01:56:08,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 01:56:18,572] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 02:01:32,244] INFO : Step 17100/50000 rank1 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2447 | Regression_cos_phi: 0.1032 | Regression_energy: 0.0810 | Regression_eta: 0.0701 | Regression_pt: 0.0842 | Regression_sin_phi: 0.1320 | LR: 7.64e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.1170s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank1 [2026-10-29 02:06:24,490] INFO : Step 17200/50000 rank1 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0707 | Regression_eta: 0.0635 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0813 | LR: 7.62e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.6100s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank1 [2026-10-29 02:11:06,514] INFO : Step 17300/50000 rank1 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0713 | Regression_eta: 0.0858 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0937 | LR: 7.59e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7534s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank1 [2026-10-29 02:16:07,513] INFO : Step 17400/50000 rank1 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2382 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0864 | Regression_eta: 0.0515 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0588 | LR: 7.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5451s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank1 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank1 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0841 | Regression_eta: 0.0558 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0602 | LR: 7.54e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2424s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank1 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank1 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1704 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0778 | Regression_eta: 0.0455 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0707 | LR: 7.52e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.4705s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank1 [2026-10-29 02:30:47,052] INFO : Step 17700/50000 rank1 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0847 | Regression_eta: 0.0708 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0894 | LR: 7.49e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7707s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank1 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank1 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0819 | Regression_eta: 0.0888 | Regression_pt: 0.0836 | Regression_sin_phi: 0.1046 | LR: 7.47e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.0198s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank1 [2026-10-29 02:40:30,746] INFO : Step 17900/50000 rank1 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0707 | Regression_eta: 0.0562 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0551 | LR: 7.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5937s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank1 [2026-10-29 02:45:16,090] INFO : Step 18000/50000 rank1 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0562 | Regression_eta: 0.0483 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0588 | LR: 7.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6580s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank1 [2026-10-29 02:45:17,189] INFO : Running validation on rank1 +rank1 [2026-10-29 02:45:17,189] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:46:12,790] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:46:13,322] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 02:46:39,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:46:40,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:46:41,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:46:42,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:46:52,552] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:46:53,608] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 02:47:09,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:47:10,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:47:12,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:47:14,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:47:24,780] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 02:47:25,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:47:25,989] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 02:47:42,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:47:43,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:47:43,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:47:44,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:47:54,787] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank1 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2323 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0674 | Regression_eta: 0.0636 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0698 | LR: 7.39e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7385s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank1 [2026-10-29 02:58:08,644] INFO : Step 18200/50000 rank1 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1941 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0871 | Regression_eta: 0.0530 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0796 | LR: 7.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9651s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank1 [2026-10-29 03:02:55,065] INFO : Step 18300/50000 rank1 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0651 | Regression_eta: 0.0582 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0743 | LR: 7.34e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank1 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank1 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0728 | Regression_eta: 0.0525 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0824 | LR: 7.31e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7129s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank1 [2026-10-29 03:12:43,564] INFO : Step 18500/50000 rank1 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2186 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0717 | Regression_eta: 0.0455 | Regression_pt: 0.0746 | Regression_sin_phi: 0.1016 | LR: 7.29e-04 | DataLoad Time: 0.0108s | Model Forward Time: 5.3972s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank1 [2026-10-29 03:17:39,474] INFO : Step 18600/50000 rank1 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0692 | Regression_eta: 0.0462 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0745 | LR: 7.26e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4811s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank1 [2026-10-29 03:22:31,157] INFO : Step 18700/50000 rank1 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0893 | Regression_eta: 0.0603 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0717 | LR: 7.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4797s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank1 [2026-10-29 03:27:37,244] INFO : Step 18800/50000 rank1 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0751 | Regression_eta: 0.0675 | Regression_pt: 0.0775 | Regression_sin_phi: 0.1778 | LR: 7.21e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.9516s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank1 [2026-10-29 03:32:26,943] INFO : Step 18900/50000 rank1 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0733 | Regression_eta: 0.0380 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0509 | LR: 7.18e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5380s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank1 [2026-10-29 03:37:06,569] INFO : Step 19000/50000 rank1 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0801 | Regression_eta: 0.0456 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0889 | LR: 7.16e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5307s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank1 [2026-10-29 03:37:07,713] INFO : Running validation on rank1 +rank1 [2026-10-29 03:37:07,713] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 03:37:07,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 03:38:00,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:38:00,821] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 03:38:26,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 03:38:28,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 03:38:29,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 03:38:30,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 03:38:40,255] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 03:38:40,770] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:38:41,425] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 03:38:57,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 03:38:58,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 03:38:59,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 03:39:00,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 03:39:10,275] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:39:11,227] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 03:39:26,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 03:39:27,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 03:39:28,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 03:39:29,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 03:39:39,121] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank1 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2100 | Regression_cos_phi: 0.1330 | Regression_energy: 0.0765 | Regression_eta: 0.0672 | Regression_pt: 0.0825 | Regression_sin_phi: 0.1222 | LR: 7.13e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank1 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank1 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0619 | Regression_eta: 0.0520 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0648 | LR: 7.10e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5604s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank1 [2026-10-29 03:54:09,158] INFO : Step 19300/50000 rank1 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0717 | Regression_eta: 0.0552 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0677 | LR: 7.08e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3945s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank1 [2026-10-29 03:58:51,216] INFO : Step 19400/50000 rank1 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0767 | Regression_eta: 0.0509 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0889 | LR: 7.05e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6150s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank1 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank1 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0744 | Regression_eta: 0.0423 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0601 | LR: 7.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7167s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank1 [2026-10-29 04:08:12,465] INFO : Step 19600/50000 rank1 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0650 | Regression_eta: 0.0362 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0561 | LR: 7.00e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.4184s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank1 [2026-10-29 04:12:52,347] INFO : Step 19700/50000 rank1 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0639 | Regression_eta: 0.0578 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0792 | LR: 6.97e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3570s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank1 [2026-10-29 04:17:28,314] INFO : Step 19800/50000 rank1 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.2020 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0697 | Regression_eta: 0.0461 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0672 | LR: 6.94e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.0465s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank1 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank1 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0872 | Regression_eta: 0.0613 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0946 | LR: 6.92e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.3169s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank1 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank1 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0575 | Regression_eta: 0.0305 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0486 | LR: 6.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6552s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank1 [2026-10-29 04:26:52,969] INFO : Running validation on rank1 +rank1 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:27:46,036] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:27:46,651] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 04:28:14,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:28:15,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:28:16,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:28:17,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:28:27,302] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:28:28,501] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 04:28:44,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:28:45,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:28:46,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:28:47,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:28:57,448] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:28:59,247] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 04:29:14,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:29:15,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:29:16,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:29:17,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:29:27,394] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 04:34:27,054] INFO : Step 20100/50000 rank1 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2390 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0745 | Regression_eta: 0.0443 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0694 | LR: 6.86e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.3247s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank1 [2026-10-29 04:39:08,147] INFO : Step 20200/50000 rank1 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0735 | Regression_eta: 0.0447 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0565 | LR: 6.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6161s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank1 [2026-10-29 04:43:47,102] INFO : Step 20300/50000 rank1 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1256 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0668 | Regression_eta: 0.0302 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0579 | LR: 6.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4958s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank1 [2026-10-29 04:48:26,732] INFO : Step 20400/50000 rank1 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0794 | Regression_eta: 0.0461 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0586 | LR: 6.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4425s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank1 [2026-10-29 04:53:08,465] INFO : Step 20500/50000 rank1 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.2223 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0621 | Regression_eta: 0.0565 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0896 | LR: 6.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0617s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank1 [2026-10-29 04:57:53,290] INFO : Step 20600/50000 rank1 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0673 | Regression_eta: 0.0403 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0788 | LR: 6.73e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7518s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank1 [2026-10-29 05:02:47,888] INFO : Step 20700/50000 rank1 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0519 | Regression_eta: 0.0754 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0908 | LR: 6.70e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6387s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank1 [2026-10-29 05:07:34,619] INFO : Step 20800/50000 rank1 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0601 | Regression_eta: 0.0454 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0629 | LR: 6.67e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.8139s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank1 [2026-10-29 05:12:18,486] INFO : Step 20900/50000 rank1 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0655 | Regression_eta: 0.0420 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0682 | LR: 6.65e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6296s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank1 [2026-10-29 05:17:03,998] INFO : Step 21000/50000 rank1 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0713 | Regression_eta: 0.0423 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0535 | LR: 6.62e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank1 [2026-10-29 05:17:04,883] INFO : Running validation on rank1 +rank1 [2026-10-29 05:17:04,883] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 05:17:04,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 05:17:58,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:17:58,806] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 05:18:22,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 05:18:23,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 05:18:24,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 05:18:25,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 05:18:35,962] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:18:37,117] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 05:18:53,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 05:18:54,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 05:18:55,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 05:18:56,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 05:19:06,150] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 05:19:06,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:19:07,255] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 05:19:22,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 05:19:23,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 05:19:24,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 05:19:25,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 05:19:35,037] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank1 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0541 | Regression_eta: 0.0475 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0806 | LR: 6.59e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.3783s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank1 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank1 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 05:29:24,522] INFO : Step 21200/50000 rank1 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1740 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0794 | Regression_eta: 0.0510 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0680 | LR: 6.56e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6426s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank1 [2026-10-29 05:34:39,116] INFO : Step 21300/50000 rank1 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0723 | Regression_eta: 0.0461 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0570 | LR: 6.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6727s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank1 [2026-10-29 05:39:29,689] INFO : Step 21400/50000 rank1 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0692 | Regression_eta: 0.0458 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0936 | LR: 6.51e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.3963s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank1 [2026-10-29 05:44:17,736] INFO : Step 21500/50000 rank1 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0466 | Regression_eta: 0.0331 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0602 | LR: 6.48e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3086s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank1 [2026-10-29 05:49:11,032] INFO : Step 21600/50000 rank1 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1037 | Regression_eta: 0.0337 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0433 | LR: 6.45e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5080s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank1 [2026-10-29 05:53:59,565] INFO : Step 21700/50000 rank1 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0689 | Regression_eta: 0.0675 | Regression_pt: 0.0723 | Regression_sin_phi: 0.1117 | LR: 6.43e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.5574s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank1 [2026-10-29 05:58:52,461] INFO : Step 21800/50000 rank1 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1850 | Regression_cos_phi: 0.1247 | Regression_energy: 0.0721 | Regression_eta: 0.0381 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0729 | LR: 6.40e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.1532s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank1 [2026-10-29 06:03:49,407] INFO : Step 21900/50000 rank1 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0712 | Regression_eta: 0.0411 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0733 | LR: 6.37e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8528s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank1 [2026-10-29 06:08:50,714] INFO : Step 22000/50000 rank1 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0800 | Regression_eta: 0.0438 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0597 | LR: 6.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.4721s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank1 [2026-10-29 06:08:51,802] INFO : Running validation on rank1 +rank1 [2026-10-29 06:08:51,802] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 06:08:51,802] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 06:09:45,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:09:46,125] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 06:10:11,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 06:10:12,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 06:10:14,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 06:10:15,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 06:10:25,476] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 06:10:26,929] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 06:10:43,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 06:10:44,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 06:10:45,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 06:10:46,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 06:10:56,700] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:10:56,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:10:57,444] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 06:11:13,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 06:11:14,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 06:11:14,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 06:11:15,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 06:11:25,831] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank1 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2179 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0704 | Regression_eta: 0.0510 | Regression_pt: 0.0742 | Regression_sin_phi: 0.1141 | LR: 6.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1895s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank1 [2026-10-29 06:21:31,723] INFO : Step 22200/50000 rank1 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0562 | Regression_eta: 0.0375 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0491 | LR: 6.29e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2935s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank1 [2026-10-29 06:26:33,413] INFO : Step 22300/50000 rank1 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0626 | Regression_eta: 0.0437 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0561 | LR: 6.26e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7520s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank1 [2026-10-29 06:31:29,807] INFO : Step 22400/50000 rank1 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0600 | Regression_eta: 0.0507 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0628 | LR: 6.23e-04 | DataLoad Time: 0.0137s | Model Forward Time: 6.4067s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank1 [2026-10-29 06:36:31,514] INFO : Step 22500/50000 rank1 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0701 | Regression_eta: 0.0357 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0679 | LR: 6.20e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.2557s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank1 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank1 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0629 | Regression_eta: 0.0639 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0676 | LR: 6.18e-04 | DataLoad Time: 0.0259s | Model Forward Time: 4.3115s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank1 [2026-10-29 06:46:29,194] INFO : Step 22700/50000 rank1 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0673 | Regression_eta: 0.0452 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0686 | LR: 6.15e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9446s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank1 [2026-10-29 06:51:26,819] INFO : Step 22800/50000 rank1 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0722 | Regression_eta: 0.0612 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0757 | LR: 6.12e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4062s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank1 [2026-10-29 06:56:23,353] INFO : Step 22900/50000 rank1 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0622 | Regression_eta: 0.0382 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0616 | LR: 6.09e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8100s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank1 [2026-10-29 07:01:33,057] INFO : Step 23000/50000 rank1 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0732 | Regression_eta: 0.0526 | Regression_pt: 0.0771 | Regression_sin_phi: 0.1050 | LR: 6.06e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.4751s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank1 [2026-10-29 07:01:34,090] INFO : Running validation on rank1 +rank1 [2026-10-29 07:01:34,090] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:01:34,090] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:02:28,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 07:02:49,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:02:50,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:02:51,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:02:52,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:03:02,572] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:03:03,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:03:03,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 07:03:19,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:03:20,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:03:21,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:03:22,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:03:32,794] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:03:33,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:03:34,171] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 07:03:49,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:03:50,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:03:51,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:03:52,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:04:02,088] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 07:09:13,971] INFO : Step 23100/50000 rank1 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0772 | Regression_eta: 0.0526 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0690 | LR: 6.04e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.1803s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank1 [2026-10-29 07:14:12,717] INFO : Step 23200/50000 rank1 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0531 | Regression_eta: 0.0394 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0609 | LR: 6.01e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6121s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank1 [2026-10-29 07:19:05,797] INFO : Step 23300/50000 rank1 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0787 | Regression_eta: 0.0316 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0720 | LR: 5.98e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8572s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank1 [2026-10-29 07:24:01,131] INFO : Step 23400/50000 rank1 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0652 | Regression_eta: 0.0362 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0771 | LR: 5.95e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0105s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank1 [2026-10-29 07:29:14,711] INFO : Step 23500/50000 rank1 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0698 | Regression_eta: 0.0435 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0862 | LR: 5.92e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.8337s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank1 [2026-10-29 07:34:22,318] INFO : Step 23600/50000 rank1 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0773 | Regression_eta: 0.0338 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0422 | LR: 5.90e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.3922s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank1 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank1 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0498 | Regression_eta: 0.0392 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0683 | LR: 5.87e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.8188s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank1 [2026-10-29 07:44:46,684] INFO : Step 23800/50000 rank1 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0834 | Regression_eta: 0.0550 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0998 | LR: 5.84e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8785s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank1 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank1 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0696 | Regression_eta: 0.0402 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0621 | LR: 5.81e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8172s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank1 [2026-10-29 07:54:58,897] INFO : Step 24000/50000 rank1 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0598 | Regression_eta: 0.0313 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0563 | LR: 5.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8547s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank1 [2026-10-29 07:54:59,831] INFO : Running validation on rank1 +rank1 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:55:53,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:55:54,113] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 07:56:20,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:56:21,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:56:22,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:56:23,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:56:33,982] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:56:35,052] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 07:56:51,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:56:52,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:56:53,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:56:54,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:57:04,014] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:57:05,694] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 07:57:21,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:57:22,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:57:23,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:57:23,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:57:33,850] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 08:03:04,652] INFO : Step 24100/50000 rank1 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0681 | Regression_eta: 0.0559 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0635 | LR: 5.75e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5905s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank1 [2026-10-29 08:07:59,793] INFO : Step 24200/50000 rank1 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0656 | Regression_eta: 0.0320 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0611 | LR: 5.73e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.0944s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank1 [2026-10-29 08:13:18,510] INFO : Step 24300/50000 rank1 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0597 | Regression_eta: 0.0435 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0984 | LR: 5.70e-04 | DataLoad Time: 0.0202s | Model Forward Time: 7.4905s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank1 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank1 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0679 | Regression_eta: 0.0668 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0910 | LR: 5.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1591s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank1 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank1 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0543 | Regression_eta: 0.0595 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0481 | LR: 5.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5209s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank1 [2026-10-29 08:28:18,514] INFO : Step 24600/50000 rank1 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0781 | Regression_eta: 0.0491 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0537 | LR: 5.61e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.7172s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank1 [2026-10-29 08:33:21,873] INFO : Step 24700/50000 rank1 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0648 | Regression_eta: 0.0517 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0630 | LR: 5.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9272s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank1 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank1 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0558 | Regression_eta: 0.0361 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0569 | LR: 5.56e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.2078s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank1 [2026-10-29 08:43:21,524] INFO : Step 24900/50000 rank1 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0556 | Regression_eta: 0.0312 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0670 | LR: 5.53e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4733s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank1 [2026-10-29 08:48:25,834] INFO : Step 25000/50000 rank1 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0262 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0602 | Regression_eta: 0.0481 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0493 | LR: 5.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9513s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank1 [2026-10-29 08:48:26,770] INFO : Running validation on rank1 +rank1 [2026-10-29 08:48:26,770] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 08:49:20,641] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:49:21,192] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 08:49:52,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 08:49:53,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 08:49:54,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 08:49:55,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 08:50:05,858] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:50:06,992] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 08:50:23,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 08:50:24,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 08:50:25,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 08:50:26,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 08:50:36,038] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:50:36,988] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 08:50:52,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 08:50:53,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 08:50:54,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 08:50:55,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 08:51:05,826] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 08:56:18,537] INFO : Step 25100/50000 rank1 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0524 | Regression_eta: 0.0433 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0789 | LR: 5.47e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.3805s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank1 [2026-10-29 09:01:01,591] INFO : Step 25200/50000 rank1 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0647 | Regression_eta: 0.0367 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0787 | LR: 5.44e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8757s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank1 [2026-10-29 09:05:50,664] INFO : Step 25300/50000 rank1 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0486 | Regression_eta: 0.0513 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0486 | LR: 5.42e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.9773s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank1 [2026-10-29 09:10:30,564] INFO : Step 25400/50000 rank1 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0657 | Regression_eta: 0.0558 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0772 | LR: 5.39e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.1704s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank1 [2026-10-29 09:15:11,239] INFO : Step 25500/50000 rank1 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0546 | Regression_eta: 0.0343 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0599 | LR: 5.36e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4711s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank1 [2026-10-29 09:19:56,597] INFO : Step 25600/50000 rank1 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0506 | Regression_eta: 0.0351 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0509 | LR: 5.33e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.9353s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank1 [2026-10-29 09:24:33,864] INFO : Step 25700/50000 rank1 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1262 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0635 | Regression_eta: 0.0379 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0502 | LR: 5.30e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2125s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank1 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank1 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0703 | Regression_eta: 0.0461 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0732 | LR: 5.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1661s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank1 [2026-10-29 09:34:19,331] INFO : Step 25900/50000 rank1 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0616 | Regression_eta: 0.0388 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0581 | LR: 5.25e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.7721s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank1 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank1 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0617 | Regression_eta: 0.0368 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0627 | LR: 5.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7494s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank1 [2026-10-29 09:39:04,382] INFO : Running validation on rank1 +rank1 [2026-10-29 09:39:04,382] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:39:04,382] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:39:55,910] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 09:40:23,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:40:24,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:40:25,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:40:26,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:40:36,431] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:40:37,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:40:37,506] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 09:40:53,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:40:55,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:40:56,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:40:57,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:41:07,402] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 09:41:08,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:44:09,566] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 09:44:24,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:44:25,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:44:26,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:44:27,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:44:37,730] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 09:50:29,554] INFO : Step 26100/50000 rank1 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0526 | Regression_eta: 0.0442 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0425 | LR: 5.19e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1768s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank1 [2026-10-29 09:55:15,273] INFO : Step 26200/50000 rank1 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0747 | Regression_eta: 0.0409 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0525 | LR: 5.16e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8149s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank1 [2026-10-29 09:59:57,745] INFO : Step 26300/50000 rank1 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0699 | Regression_eta: 0.0461 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0477 | LR: 5.13e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.9314s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank1 [2026-10-29 10:04:36,479] INFO : Step 26400/50000 rank1 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0530 | Regression_eta: 0.0263 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0608 | LR: 5.10e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.1474s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank1 [2026-10-29 10:09:23,740] INFO : Step 26500/50000 rank1 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0572 | Regression_eta: 0.0415 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0501 | LR: 5.08e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.0956s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank1 [2026-10-29 10:14:05,781] INFO : Step 26600/50000 rank1 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1264 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0766 | Regression_eta: 0.0285 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0361 | LR: 5.05e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6626s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank1 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank1 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0605 | Regression_eta: 0.0306 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0474 | LR: 5.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2154s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank1 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank1 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1870 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0486 | Regression_eta: 0.0497 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0862 | LR: 4.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5308s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank1 [2026-10-29 10:28:51,386] INFO : Step 26900/50000 rank1 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0768 | Regression_eta: 0.0278 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0568 | LR: 4.96e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.5623s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank1 [2026-10-29 10:33:37,219] INFO : Step 27000/50000 rank1 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0644 | Regression_eta: 0.0480 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0741 | LR: 4.94e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.7006s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank1 [2026-10-29 10:33:38,214] INFO : Running validation on rank1 +rank1 [2026-10-29 10:33:38,214] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 10:33:38,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 10:34:33,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:34:34,224] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 10:35:00,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 10:35:01,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 10:35:02,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 10:35:03,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 10:35:13,147] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:35:13,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:35:14,236] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 10:35:29,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 10:35:31,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 10:35:32,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 10:35:33,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 10:35:43,208] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:35:44,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:35:45,038] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 10:36:00,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 10:36:01,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 10:36:02,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 10:36:03,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 10:36:13,094] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 10:41:28,284] INFO : Step 27100/50000 rank1 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1004 | Regression_energy: 0.0838 | Regression_eta: 0.0581 | Regression_pt: 0.0848 | Regression_sin_phi: 0.1038 | LR: 4.91e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6744s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank1 [2026-10-29 10:46:22,294] INFO : Step 27200/50000 rank1 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0644 | Regression_eta: 0.0702 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0821 | LR: 4.88e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8257s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank1 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank1 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0282 | Classification_binary: 0.1043 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0495 | Regression_eta: 0.0247 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0593 | LR: 4.85e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6696s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank1 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank1 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0792 | Regression_eta: 0.0381 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0842 | LR: 4.82e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9754s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank1 [2026-10-29 11:01:35,254] INFO : Step 27500/50000 rank1 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0537 | Regression_eta: 0.0313 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0502 | LR: 4.80e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.3147s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank1 [2026-10-29 11:08:29,070] INFO : Step 27600/50000 rank1 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0488 | Regression_eta: 0.0294 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0491 | LR: 4.77e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.3316s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank1 [2026-10-29 11:13:33,057] INFO : Step 27700/50000 rank1 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1118 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0602 | Regression_eta: 0.0271 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0500 | LR: 4.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7949s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank1 [2026-10-29 11:18:30,825] INFO : Step 27800/50000 rank1 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.2114 | Regression_cos_phi: 0.1220 | Regression_energy: 0.0556 | Regression_eta: 0.0535 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0693 | LR: 4.71e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.0104s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank1 [2026-10-29 11:23:38,362] INFO : Step 27900/50000 rank1 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0688 | Regression_eta: 0.0481 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0797 | LR: 4.68e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8883s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank1 [2026-10-29 11:28:54,872] INFO : Step 28000/50000 rank1 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1226 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0677 | Regression_eta: 0.0339 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0603 | LR: 4.66e-04 | DataLoad Time: 0.0226s | Model Forward Time: 5.9076s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank1 [2026-10-29 11:28:55,939] INFO : Running validation on rank1 +rank1 [2026-10-29 11:28:55,939] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 11:28:55,939] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 11:29:47,971] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:29:48,524] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 11:30:14,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 11:30:15,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 11:30:16,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 11:30:17,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 11:30:27,545] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 11:30:28,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:30:28,727] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 11:30:44,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 11:30:45,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 11:30:46,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 11:30:47,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 11:30:57,565] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 11:30:57,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 11:31:14,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 11:31:14,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 11:31:15,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 11:31:16,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 11:31:26,691] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 11:37:01,722] INFO : Step 28100/50000 rank1 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0547 | Regression_eta: 0.0497 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0888 | LR: 4.63e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.5019s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank1 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank1 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0515 | Regression_eta: 0.0399 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0614 | LR: 4.60e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.0742s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank1 [2026-10-29 11:47:09,284] INFO : Step 28300/50000 rank1 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0586 | Regression_eta: 0.0501 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0866 | LR: 4.57e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.1007s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank1 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank1 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1634 | Classification_binary: 0.2801 | Regression_cos_phi: 0.0802 | Regression_energy: 0.1406 | Regression_eta: 0.0742 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0783 | LR: 4.55e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5907s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank1 [2026-10-29 11:56:56,842] INFO : Step 28500/50000 rank1 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0427 | Regression_eta: 0.0455 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0642 | LR: 4.52e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.9935s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank1 [2026-10-29 12:01:46,924] INFO : Step 28600/50000 rank1 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0617 | Regression_eta: 0.0353 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0687 | LR: 4.49e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6592s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank1 [2026-10-29 12:06:39,994] INFO : Step 28700/50000 rank1 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1220 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0480 | Regression_eta: 0.0313 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0385 | LR: 4.46e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5152s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank1 [2026-10-29 12:11:31,029] INFO : Step 28800/50000 rank1 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0488 | Regression_eta: 0.0370 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0670 | LR: 4.44e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.9314s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank1 [2026-10-29 12:16:17,097] INFO : Step 28900/50000 rank1 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0524 | Regression_eta: 0.0408 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0551 | LR: 4.41e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6992s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank1 [2026-10-29 12:21:10,133] INFO : Step 29000/50000 rank1 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0618 | Regression_eta: 0.0480 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0557 | LR: 4.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9917s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank1 [2026-10-29 12:21:11,218] INFO : Running validation on rank1 +rank1 [2026-10-29 12:21:11,218] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 12:21:11,218] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 12:22:04,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 12:22:31,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 12:22:32,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 12:22:33,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 12:22:34,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 12:22:44,736] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:22:45,974] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 12:23:02,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 12:23:02,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 12:23:03,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 12:23:04,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 12:23:14,937] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 12:23:15,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:23:16,444] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 12:23:32,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 12:23:33,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 12:23:33,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 12:23:34,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 12:23:44,801] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank1 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0567 | Regression_eta: 0.0367 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0811 | LR: 4.35e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1713s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank1 [2026-10-29 12:33:43,193] INFO : Step 29200/50000 rank1 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1991 | Regression_cos_phi: 0.1158 | Regression_energy: 0.0539 | Regression_eta: 0.0602 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0745 | LR: 4.33e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.5643s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank1 [2026-10-29 14:49:26,814] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-29 14:49:26,831] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank1 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-29 14:49:54,088] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank1 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank1 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank1 [2026-10-29 14:49:54,133] INFO : Initializing backbone layer 2 +rank1 [2026-10-29 14:49:54,141] INFO : Initializing backbone layer 3 +rank1 [2026-10-29 14:49:54,150] INFO : Initializing backbone layer 4 +rank1 [2026-10-29 14:49:54,158] INFO : Initializing backbone layer 5 +rank1 [2026-10-29 14:49:54,167] INFO : Backbone initialization took 0.08s +rank1 [2026-10-29 14:49:54,167] INFO : backbone parameters: 7901376 +rank1 [2026-10-29 14:49:54,167] INFO : Initializing output DNNs +rank1 [2026-10-29 14:49:54,203] INFO : Output DNNs initialization took 0.04s +rank1 [2026-10-29 14:49:54,203] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-29 14:49:54,203] INFO : jagged_attention enabled=False +rank1 [2026-10-29 14:49:54,203] INFO : flash_attn_varlen enabled=False +rank1 [2026-10-29 14:49:54,203] INFO : task_queries enabled=False +rank1 [2026-10-29 14:49:54,203] INFO : classification_norm parameters: 0 +rank1 [2026-10-29 14:49:54,203] INFO : regression_norm parameters: 0 +rank1 [2026-10-29 14:49:54,203] INFO : classification_query parameters: 0 +rank1 [2026-10-29 14:49:54,203] INFO : regression_query parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : classification_readout parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : regression_readout parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : output_mode=set +rank1 [2026-10-29 14:49:54,204] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-29 14:49:54,204] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_pid parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_pt parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_eta parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-29 14:49:54,204] INFO : nn_energy parameters: 0 +rank1 [2026-10-29 14:49:54,205] INFO : Total MLPF parameters: 12786964 +rank1 [2026-10-29 14:49:54,205] INFO : MLPF __init__ done +rank1 [2026-10-29 14:49:54,205] INFO : Instantiated model +rank1 [2026-10-29 14:49:54,205] INFO : Moving model to device rank=1 +rank1 [2026-10-29 14:49:54,226] INFO : Moved model to device rank=1 +rank1 [2026-10-29 14:49:54,228] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-29 14:49:54,790] INFO : Restoring training from step 29001 +rank1 [2026-10-29 14:49:54,797] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank1 [2026-10-29 14:49:54,798] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-29 14:49:55,359] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-29 14:49:55,510] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank1 [2026-10-29 14:49:55,510] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-29 14:49:55,512] INFO : Trainable parameters: 12786964 +rank1 [2026-10-29 14:49:55,512] INFO : Non-trainable parameters: 23268 +rank1 [2026-10-29 14:49:55,512] INFO : Total parameters: 12810232 +rank1 [2026-10-29 14:49:55,515] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-29 14:49:55,516] INFO : Getting dataloaders +rank1 [2026-10-29 14:49:55,516] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:01,367] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:02,348] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:03,544] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-29 14:50:03,544] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-29 14:50:03,544] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:03,812] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:04,087] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:50:04,339] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-29 14:50:04,339] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-29 14:50:18,249] INFO : Got dataloaders +rank1 [2026-10-29 14:50:18,249] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank1 [2026-10-29 14:50:18,249] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank1 [2026-10-29 14:50:18,249] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank1 [2026-10-29 14:50:18,249] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank1 [2026-10-29 14:50:19,866] INFO : loader split=train rank=1 len=21108 +rank1 [2026-10-29 14:50:19,866] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-29 14:50:19,866] INFO : Starting training from step 29001 to 50000 on rank 1 of 8 +rank1 [2026-10-29 14:50:19,866] INFO : Creating train_iterator +rank1 [2026-10-29 14:50:19,866] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank1 [2026-10-29 14:50:33,975] INFO : Created train_iterator +rank1 [2026-10-29 14:55:12,919] INFO : Step 29100/50000 rank1 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0529 | Regression_eta: 0.0391 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0588 | LR: 4.35e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7152s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank1 [2026-10-29 14:59:46,789] INFO : Step 29200/50000 rank1 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1965 | Regression_cos_phi: 0.1106 | Regression_energy: 0.0568 | Regression_eta: 0.0558 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0760 | LR: 4.33e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3628s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank1 [2026-10-29 15:04:21,834] INFO : Step 29300/50000 rank1 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0625 | Regression_eta: 0.0297 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0520 | LR: 4.30e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0917s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank1 [2026-10-29 15:08:57,030] INFO : Step 29400/50000 rank1 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0499 | Regression_eta: 0.0319 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0434 | LR: 4.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2723s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank1 [2026-10-29 15:13:31,010] INFO : Step 29500/50000 rank1 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1797 | Regression_cos_phi: 0.1057 | Regression_energy: 0.0553 | Regression_eta: 0.0619 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0849 | LR: 4.24e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4981s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank1 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank1 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0576 | Regression_eta: 0.0379 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0534 | LR: 4.22e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5925s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank1 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank1 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0432 | Regression_eta: 0.0257 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0401 | LR: 4.19e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6715s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank1 [2026-10-29 15:27:18,562] INFO : Step 29800/50000 rank1 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0655 | Regression_eta: 0.0560 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0792 | LR: 4.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6296s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank1 [2026-10-29 15:31:51,911] INFO : Step 29900/50000 rank1 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0542 | Regression_eta: 0.0428 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0599 | LR: 4.14e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5841s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank1 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank1 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0533 | Regression_eta: 0.0353 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0467 | LR: 4.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4519s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank1 [2026-10-29 15:36:27,738] INFO : Running validation on rank1 +rank1 [2026-10-29 15:36:27,738] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 15:37:19,265] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 15:37:50,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 15:37:51,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 15:37:52,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 15:37:53,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 15:38:03,920] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:38:04,946] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 15:38:20,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 15:38:21,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 15:38:22,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 15:38:23,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 15:38:33,755] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:38:35,047] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 15:38:50,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 15:38:50,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 15:38:51,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 15:38:52,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 15:39:02,611] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank1 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0586 | Regression_eta: 0.0410 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0596 | LR: 4.08e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7107s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank1 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank1 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0517 | Regression_eta: 0.0505 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0988 | LR: 4.06e-04 | DataLoad Time: 0.0268s | Model Forward Time: 4.7628s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank1 [2026-10-29 15:53:38,867] INFO : Step 30300/50000 rank1 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0278 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0527 | Regression_eta: 0.0386 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0531 | LR: 4.03e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3375s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank1 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank1 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0626 | Regression_eta: 0.0684 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0452 | LR: 4.00e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4506s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank1 [2026-10-29 16:02:51,486] INFO : Step 30500/50000 rank1 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2216 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0703 | Regression_eta: 0.0399 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0659 | LR: 3.98e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4573s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank1 [2026-10-29 16:07:32,050] INFO : Step 30600/50000 rank1 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0623 | Regression_eta: 0.0533 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0536 | LR: 3.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9377s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank1 [2026-10-29 16:12:18,780] INFO : Step 30700/50000 rank1 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1171 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0558 | Regression_eta: 0.0295 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0520 | LR: 3.92e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2824s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank1 [2026-10-29 16:17:03,635] INFO : Step 30800/50000 rank1 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0615 | Regression_eta: 0.0401 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0626 | LR: 3.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0634s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank1 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank1 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0541 | Regression_eta: 0.0348 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0384 | LR: 3.87e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6929s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank1 [2026-10-29 16:26:02,768] INFO : Step 31000/50000 rank1 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0546 | Regression_eta: 0.0585 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0605 | LR: 3.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0248s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank1 [2026-10-29 16:26:49,212] INFO : Running validation on rank1 +rank1 [2026-10-29 16:26:49,213] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:26:49,213] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:27:39,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:27:40,304] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 16:28:08,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:28:10,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:28:12,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:28:14,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:28:24,165] INFO : Time taken to make predictions on device 1 is: 0.31 min +rank1 [2026-10-29 16:28:24,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:28:24,725] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 16:28:40,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:28:41,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:28:42,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:28:43,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:28:53,225] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:28:54,191] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:28:54,676] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 16:29:10,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:29:11,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:29:11,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:29:12,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:29:22,760] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank1 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0642 | Regression_eta: 0.0678 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0755 | LR: 3.82e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5514s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank1 [2026-10-29 16:39:13,624] INFO : Step 31200/50000 rank1 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0493 | Regression_eta: 0.0404 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0423 | LR: 3.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0725s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank1 [2026-10-29 16:43:57,174] INFO : Step 31300/50000 rank1 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0651 | Regression_eta: 0.0447 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0653 | LR: 3.76e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8832s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank1 [2026-10-29 16:48:30,666] INFO : Step 31400/50000 rank1 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1419 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0470 | Regression_eta: 0.0342 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0877 | LR: 3.74e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7836s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank1 [2026-10-29 16:53:07,751] INFO : Step 31500/50000 rank1 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0666 | Regression_eta: 0.0472 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0427 | LR: 3.71e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4917s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank1 [2026-10-29 16:57:38,607] INFO : Step 31600/50000 rank1 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0457 | Regression_eta: 0.0336 | Regression_pt: 0.0461 | Regression_sin_phi: 0.1189 | LR: 3.69e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1663s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank1 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank1 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0654 | Regression_eta: 0.0399 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0835 | LR: 3.66e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1013s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank1 [2026-10-29 17:06:47,525] INFO : Step 31800/50000 rank1 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0495 | Regression_eta: 0.0407 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0460 | LR: 3.64e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2359s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank1 [2026-10-29 17:11:27,372] INFO : Step 31900/50000 rank1 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0322 | Regression_energy: 0.0692 | Regression_eta: 0.0283 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0681 | LR: 3.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5192s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank1 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank1 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0672 | Regression_eta: 0.0449 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0489 | LR: 3.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4954s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank1 [2026-10-29 17:16:12,934] INFO : Running validation on rank1 +rank1 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 17:17:03,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:17:04,363] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 17:17:32,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 17:17:33,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 17:17:34,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 17:17:35,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 17:17:45,221] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:17:46,319] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 17:18:01,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 17:18:02,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 17:18:03,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 17:18:04,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 17:18:14,856] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:18:15,920] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:18:16,468] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 17:18:32,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 17:18:32,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 17:18:33,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 17:18:34,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 17:18:44,633] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 17:23:51,375] INFO : Step 32100/50000 rank1 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1136 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0468 | Regression_eta: 0.0387 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0501 | LR: 3.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9458s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank1 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank1 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0607 | Regression_eta: 0.0568 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0623 | LR: 3.53e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4542s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank1 [2026-10-29 17:33:21,420] INFO : Step 32300/50000 rank1 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0638 | Regression_eta: 0.0355 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0505 | LR: 3.51e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.9296s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank1 [2026-10-29 17:38:08,350] INFO : Step 32400/50000 rank1 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0508 | Regression_eta: 0.0471 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0508 | LR: 3.48e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8155s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank1 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank1 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0541 | Regression_eta: 0.0368 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0557 | LR: 3.46e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8577s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank1 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank1 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0679 | Regression_eta: 0.0369 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0911 | LR: 3.43e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6209s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank1 [2026-10-29 17:52:17,188] INFO : Step 32700/50000 rank1 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0585 | Regression_eta: 0.0323 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0585 | LR: 3.41e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5867s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank1 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank1 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0444 | Regression_eta: 0.0534 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0590 | LR: 3.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5884s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank1 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank1 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0551 | Regression_eta: 0.0420 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0728 | LR: 3.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4256s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank1 [2026-10-29 18:06:22,610] INFO : Step 33000/50000 rank1 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0648 | Regression_eta: 0.0295 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0391 | LR: 3.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2108s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank1 [2026-10-29 18:06:23,516] INFO : Running validation on rank1 +rank1 [2026-10-29 18:06:23,516] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:06:23,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:07:15,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:07:15,765] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 18:07:43,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:07:44,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:07:45,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:07:47,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:07:57,020] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:07:57,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:07:58,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 18:08:14,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:08:15,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:08:16,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:08:17,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:08:27,408] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:08:28,344] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 18:08:43,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:08:44,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:08:45,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:08:46,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:08:56,200] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 18:13:51,312] INFO : Step 33100/50000 rank1 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0470 | Regression_eta: 0.0340 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0568 | LR: 3.31e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2594s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank1 [2026-10-29 18:18:33,980] INFO : Step 33200/50000 rank1 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0515 | Regression_eta: 0.0359 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0570 | LR: 3.28e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4338s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank1 [2026-10-29 18:23:15,420] INFO : Step 33300/50000 rank1 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0585 | Regression_eta: 0.0277 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0634 | LR: 3.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6019s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank1 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank1 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.0954 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0525 | Regression_eta: 0.0297 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0592 | LR: 3.23e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.2639s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank1 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank1 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1704 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0592 | Regression_eta: 0.0445 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0804 | LR: 3.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3725s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank1 [2026-10-29 18:37:27,752] INFO : Step 33600/50000 rank1 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0589 | Regression_eta: 0.0406 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0491 | LR: 3.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3461s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank1 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank1 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0567 | Regression_eta: 0.0410 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0716 | LR: 3.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3198s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank1 [2026-10-29 18:47:04,986] INFO : Step 33800/50000 rank1 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0480 | Regression_eta: 0.0369 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0568 | LR: 3.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3941s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank1 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank1 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0223 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0507 | Regression_eta: 0.0341 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0670 | LR: 3.11e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1945s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank1 [2026-10-29 18:56:35,807] INFO : Step 34000/50000 rank1 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0537 | Regression_eta: 0.0301 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0607 | LR: 3.09e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1631s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank1 [2026-10-29 18:56:36,806] INFO : Running validation on rank1 +rank1 [2026-10-29 18:56:36,806] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:56:36,806] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:57:29,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:57:30,357] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 18:57:54,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:57:55,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:57:57,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:57:58,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:58:08,056] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:58:08,732] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:58:09,211] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 18:58:24,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:58:25,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:58:26,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:58:27,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:58:37,824] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:58:39,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 18:58:55,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:58:55,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:58:56,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:58:57,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:59:07,577] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 19:04:16,498] INFO : Step 34100/50000 rank1 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1576 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0528 | Regression_eta: 0.0327 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0693 | LR: 3.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8507s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank1 [2026-10-29 19:09:14,636] INFO : Step 34200/50000 rank1 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0805 | Regression_eta: 0.0397 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0527 | LR: 3.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6992s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank1 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank1 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0640 | Regression_eta: 0.0467 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0797 | LR: 3.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3187s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank1 [2026-10-29 19:18:48,203] INFO : Step 34400/50000 rank1 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0647 | Regression_eta: 0.0176 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0281 | LR: 2.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9230s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank1 [2026-10-29 19:23:49,388] INFO : Step 34500/50000 rank1 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1790 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0514 | Regression_eta: 0.0380 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0630 | LR: 2.97e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2310s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank1 [2026-10-29 19:28:34,775] INFO : Step 34600/50000 rank1 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0364 | Regression_energy: 0.0456 | Regression_eta: 0.0341 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0669 | LR: 2.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4735s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank1 [2026-10-29 19:33:35,743] INFO : Step 34700/50000 rank1 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0391 | Regression_eta: 0.0263 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0562 | LR: 2.92e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1486s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank1 [2026-10-29 19:38:26,588] INFO : Step 34800/50000 rank1 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0544 | Regression_eta: 0.0312 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0337 | LR: 2.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0545s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank1 [2026-10-29 19:43:15,573] INFO : Step 34900/50000 rank1 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0481 | Regression_eta: 0.0457 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0638 | LR: 2.88e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2755s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank1 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank1 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1151 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0524 | Regression_eta: 0.0383 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0559 | LR: 2.85e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3857s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank1 [2026-10-29 19:48:21,334] INFO : Running validation on rank1 +rank1 [2026-10-29 19:48:21,334] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 19:48:21,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 19:49:15,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:49:15,988] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 19:49:42,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 19:49:43,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 19:49:44,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 19:49:45,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 19:49:55,640] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:49:56,263] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:49:56,737] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 19:50:12,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 19:50:13,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 19:50:14,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 19:50:15,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 19:50:25,148] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:50:26,882] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 19:50:42,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 19:50:43,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 19:50:44,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 19:50:44,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 19:50:54,918] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 19:56:07,675] INFO : Step 35100/50000 rank1 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0486 | Regression_eta: 0.0557 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0891 | LR: 2.83e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4688s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank1 [2026-10-29 20:01:01,618] INFO : Step 35200/50000 rank1 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0443 | Regression_eta: 0.0296 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0391 | LR: 2.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6697s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank1 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank1 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0495 | Regression_eta: 0.0282 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0595 | LR: 2.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1453s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank1 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank1 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0617 | Regression_eta: 0.0462 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0475 | LR: 2.76e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4649s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank1 [2026-10-29 20:15:42,288] INFO : Step 35500/50000 rank1 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0467 | Regression_eta: 0.0258 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0424 | LR: 2.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3575s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank1 [2026-10-29 20:20:38,046] INFO : Step 35600/50000 rank1 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0477 | Regression_eta: 0.0411 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0641 | LR: 2.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1903s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank1 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank1 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0386 | Regression_eta: 0.0351 | Regression_pt: 0.0394 | Regression_sin_phi: 0.0709 | LR: 2.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1725s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank1 [2026-10-29 20:30:22,091] INFO : Step 35800/50000 rank1 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0467 | Regression_eta: 0.0337 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0453 | LR: 2.68e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2812s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank1 [2026-10-29 20:35:20,188] INFO : Step 35900/50000 rank1 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0516 | Regression_eta: 0.0379 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0495 | LR: 2.65e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6689s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank1 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank1 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0559 | Regression_eta: 0.0347 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0493 | LR: 2.63e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2578s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank1 [2026-10-29 20:40:17,058] INFO : Running validation on rank1 +rank1 [2026-10-29 20:40:17,058] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 20:40:17,058] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 20:41:08,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:41:08,959] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 20:41:36,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 20:41:37,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 20:41:38,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 20:41:39,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 20:41:49,085] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:41:49,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:41:50,340] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 20:42:06,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 20:42:07,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 20:42:08,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 20:42:09,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 20:42:19,064] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:42:20,694] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 20:42:36,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 20:42:36,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 20:42:37,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 20:42:38,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 20:42:48,641] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 20:47:46,988] INFO : Step 36100/50000 rank1 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1101 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0532 | Regression_eta: 0.0259 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0496 | LR: 2.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4491s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank1 [2026-10-29 20:52:22,841] INFO : Step 36200/50000 rank1 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0514 | Regression_eta: 0.0456 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0922 | LR: 2.59e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3972s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank1 [2026-10-29 20:56:59,585] INFO : Step 36300/50000 rank1 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0518 | Regression_eta: 0.0377 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0452 | LR: 2.57e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2272s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank1 [2026-10-29 21:01:38,351] INFO : Step 36400/50000 rank1 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1144 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0442 | Regression_eta: 0.0344 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0499 | LR: 2.55e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8764s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank1 [2026-10-29 21:06:14,246] INFO : Step 36500/50000 rank1 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0482 | Regression_eta: 0.0290 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0442 | LR: 2.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4223s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank1 [2026-10-29 21:10:49,731] INFO : Step 36600/50000 rank1 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0355 | Regression_eta: 0.0380 | Regression_pt: 0.0358 | Regression_sin_phi: 0.0448 | LR: 2.50e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2756s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank1 [2026-10-29 21:15:35,108] INFO : Step 36700/50000 rank1 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0253 | Classification_binary: 0.1019 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0514 | Regression_eta: 0.0494 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0689 | LR: 2.48e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2692s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank1 [2026-10-29 21:20:28,003] INFO : Step 36800/50000 rank1 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0604 | Regression_eta: 0.0407 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0579 | LR: 2.46e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3994s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank1 [2026-10-29 21:25:23,681] INFO : Step 36900/50000 rank1 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0469 | Regression_eta: 0.0346 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0474 | LR: 2.44e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9350s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank1 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank1 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0443 | Regression_eta: 0.0314 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0452 | LR: 2.42e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.9320s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank1 [2026-10-29 21:30:17,322] INFO : Running validation on rank1 +rank1 [2026-10-29 21:30:17,322] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:30:17,322] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:31:08,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:31:08,621] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 21:31:35,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:31:36,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:31:37,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:31:38,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:31:48,495] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:31:49,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 21:32:05,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:32:06,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:32:07,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:32:08,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:32:18,448] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:32:20,811] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 21:32:36,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:32:37,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:32:37,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:32:38,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:32:48,768] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 21:37:56,197] INFO : Step 37100/50000 rank1 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0530 | Regression_eta: 0.0382 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0687 | LR: 2.40e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.6900s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank1 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank1 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0436 | Regression_eta: 0.0271 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0403 | LR: 2.38e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6948s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank1 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank1 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0607 | Regression_eta: 0.0375 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0533 | LR: 2.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5583s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank1 [2026-10-29 21:52:56,910] INFO : Step 37400/50000 rank1 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0409 | Regression_eta: 0.0367 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0436 | LR: 2.34e-04 | DataLoad Time: 0.0114s | Model Forward Time: 4.6282s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank1 [2026-10-29 21:57:43,877] INFO : Step 37500/50000 rank1 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0420 | Regression_eta: 0.0340 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0663 | LR: 2.32e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7100s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank1 [2026-10-29 22:02:29,841] INFO : Step 37600/50000 rank1 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0539 | Regression_eta: 0.0492 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0564 | LR: 2.30e-04 | DataLoad Time: 0.0134s | Model Forward Time: 4.2500s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank1 [2026-10-29 22:07:05,610] INFO : Step 37700/50000 rank1 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0486 | Regression_eta: 0.0772 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0382 | LR: 2.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1567s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank1 [2026-10-29 22:11:38,775] INFO : Step 37800/50000 rank1 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0561 | Regression_eta: 0.0575 | Regression_pt: 0.0579 | Regression_sin_phi: 0.1100 | LR: 2.26e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2575s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank1 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank1 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0480 | Regression_eta: 0.0281 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0726 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1265s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank1 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank1 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0576 | Regression_eta: 0.0378 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0437 | LR: 2.22e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2893s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank1 [2026-10-29 22:20:57,875] INFO : Running validation on rank1 +rank1 [2026-10-29 22:20:57,875] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 22:21:48,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:21:48,922] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 22:22:15,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 22:22:16,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 22:22:17,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 22:22:18,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 22:22:28,209] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 22:22:28,859] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:22:29,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 22:22:44,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 22:22:45,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 22:22:46,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 22:22:47,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 22:22:57,852] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:22:59,394] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 22:23:14,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 22:23:15,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 22:23:16,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 22:23:17,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 22:23:27,022] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank1 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0623 | Regression_eta: 0.0341 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0511 | LR: 2.20e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2304s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank1 [2026-10-29 22:33:07,240] INFO : Step 38200/50000 rank1 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0385 | Regression_eta: 0.0520 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0705 | LR: 2.18e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7608s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank1 [2026-10-29 22:37:42,242] INFO : Step 38300/50000 rank1 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0651 | Regression_eta: 0.0372 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0569 | LR: 2.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5937s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank1 [2026-10-29 22:42:17,608] INFO : Step 38400/50000 rank1 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0599 | Regression_eta: 0.0363 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0526 | LR: 2.14e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4882s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank1 [2026-10-29 22:46:52,192] INFO : Step 38500/50000 rank1 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1239 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0559 | Regression_eta: 0.0346 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0323 | LR: 2.12e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5213s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank1 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank1 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0395 | Regression_eta: 0.0356 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0526 | LR: 2.11e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7260s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank1 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank1 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0407 | Regression_eta: 0.0298 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0435 | LR: 2.09e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1562s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank1 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank1 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1078 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0522 | Regression_eta: 0.0257 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0593 | LR: 2.07e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3981s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank1 [2026-10-29 23:07:29,432] INFO : Step 38900/50000 rank1 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0474 | Regression_eta: 0.0276 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0452 | LR: 2.05e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8954s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank1 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank1 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0312 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0564 | Regression_eta: 0.0413 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0392 | LR: 2.03e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4692s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank1 [2026-10-29 23:12:38,865] INFO : Running validation on rank1 +rank1 [2026-10-29 23:12:38,865] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 23:12:38,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 23:13:33,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:13:34,228] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-29 23:14:06,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 23:14:07,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 23:14:08,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 23:14:09,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 23:14:19,461] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 23:14:20,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:14:20,782] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-29 23:14:36,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 23:14:37,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 23:14:38,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 23:14:39,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 23:14:49,418] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:14:50,898] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-29 23:15:06,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 23:15:07,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 23:15:07,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 23:15:08,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 23:15:18,792] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 23:20:33,571] INFO : Step 39100/50000 rank1 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0513 | Regression_eta: 0.0198 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0397 | LR: 2.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank1 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank1 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0449 | Regression_eta: 0.0321 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0466 | LR: 2.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5905s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank1 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank1 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0471 | Regression_eta: 0.0322 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0366 | LR: 1.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6308s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank1 [2026-10-29 23:34:45,054] INFO : Step 39400/50000 rank1 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1235 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0389 | Regression_eta: 0.0295 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0465 | LR: 1.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6960s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank1 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank1 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0546 | Regression_eta: 0.0366 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0353 | LR: 1.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4400s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank1 [2026-10-29 23:43:57,880] INFO : Step 39600/50000 rank1 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0481 | Regression_eta: 0.0262 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0424 | LR: 1.93e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4693s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank1 [2026-10-29 23:48:42,956] INFO : Step 39700/50000 rank1 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0506 | Regression_eta: 0.0392 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0726 | LR: 1.91e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4028s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank1 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank1 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1079 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0516 | Regression_eta: 0.0213 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0266 | LR: 1.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5380s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank1 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank1 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0449 | Regression_eta: 0.0317 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0366 | LR: 1.88e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7654s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank1 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank1 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0530 | Regression_eta: 0.0419 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0485 | LR: 1.86e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9285s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank1 [2026-10-30 00:05:56,735] INFO : Running validation on rank1 +rank1 [2026-10-30 00:05:56,735] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:05:56,735] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:06:50,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:06:51,354] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 00:07:17,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:07:18,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:07:19,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:07:20,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:07:30,823] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:07:31,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:07:32,020] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 00:07:47,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:07:48,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:07:49,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:07:50,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:08:00,788] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:08:01,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 00:08:17,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:08:18,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 00:08:19,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 00:08:19,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 00:08:29,908] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank1 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0520 | Regression_eta: 0.0316 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0444 | LR: 1.84e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6410s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank1 [2026-10-30 00:18:43,887] INFO : Step 40200/50000 rank1 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0352 | Regression_eta: 0.0476 | Regression_pt: 0.0360 | Regression_sin_phi: 0.0671 | LR: 1.83e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4446s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank1 [2026-10-30 00:23:35,702] INFO : Step 40300/50000 rank1 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0438 | Regression_eta: 0.0340 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0439 | LR: 1.81e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5322s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank1 [2026-10-30 00:28:38,368] INFO : Step 40400/50000 rank1 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0281 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0316 | Regression_energy: 0.0346 | Regression_eta: 0.0273 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0439 | LR: 1.79e-04 | DataLoad Time: 0.0108s | Model Forward Time: 4.2517s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank1 [2026-10-30 00:33:25,153] INFO : Step 40500/50000 rank1 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1215 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0492 | Regression_eta: 0.0352 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0462 | LR: 1.78e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2279s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank1 [2026-10-30 00:38:07,832] INFO : Step 40600/50000 rank1 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0457 | Regression_eta: 0.0312 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0510 | LR: 1.76e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.2984s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank1 [2026-10-30 00:42:54,582] INFO : Step 40700/50000 rank1 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0413 | Regression_eta: 0.0264 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0433 | LR: 1.75e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2542s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank1 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank1 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.0907 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0444 | Regression_eta: 0.0325 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0393 | LR: 1.73e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8886s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank1 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank1 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0366 | Regression_eta: 0.0272 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0430 | LR: 1.72e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.7063s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank1 [2026-10-30 00:57:38,928] INFO : Step 41000/50000 rank1 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0231 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0501 | Regression_eta: 0.0227 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0468 | LR: 1.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3864s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank1 [2026-10-30 00:57:39,930] INFO : Running validation on rank1 +rank1 [2026-10-30 00:57:39,930] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:57:39,930] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:58:31,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:58:32,295] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 00:58:59,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:59:00,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:59:01,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:59:02,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:59:12,421] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:59:13,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:59:13,671] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 00:59:29,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:59:30,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:59:31,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:59:32,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:59:42,220] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:59:43,889] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 00:59:59,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:59:59,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:00:00,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:00:01,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:00:11,660] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 01:05:10,996] INFO : Step 41100/50000 rank1 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0542 | Regression_eta: 0.0249 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0541 | LR: 1.69e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7656s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank1 [2026-10-30 01:10:09,652] INFO : Step 41200/50000 rank1 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.0910 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0407 | Regression_eta: 0.0335 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0446 | LR: 1.67e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6353s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank1 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank1 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0522 | Regression_eta: 0.0250 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0528 | LR: 1.66e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2635s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank1 [2026-10-30 01:19:55,521] INFO : Step 41400/50000 rank1 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0371 | Regression_eta: 0.0237 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0473 | LR: 1.64e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.6734s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank1 [2026-10-30 01:24:55,601] INFO : Step 41500/50000 rank1 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1104 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0431 | Regression_eta: 0.0292 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0468 | LR: 1.63e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.4495s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank1 [2026-10-30 01:29:39,759] INFO : Step 41600/50000 rank1 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0417 | Regression_eta: 0.0393 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0627 | LR: 1.61e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.0141s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank1 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank1 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0491 | Regression_eta: 0.0293 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0330 | LR: 1.60e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7032s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank1 [2026-10-30 01:39:10,939] INFO : Step 41800/50000 rank1 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1221 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0606 | Regression_eta: 0.0255 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0522 | LR: 1.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8797s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank1 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank1 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0212 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0486 | Regression_eta: 0.0199 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0509 | LR: 1.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2464s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank1 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank1 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0386 | Regression_eta: 0.0331 | Regression_pt: 0.0393 | Regression_sin_phi: 0.0326 | LR: 1.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2704s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank1 [2026-10-30 01:49:03,868] INFO : Running validation on rank1 +rank1 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 01:49:57,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 01:50:24,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 01:50:26,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 01:50:27,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 01:50:28,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 01:50:38,108] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:50:39,487] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 01:50:55,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 01:50:56,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 01:50:57,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 01:50:58,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 01:51:08,588] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:51:10,100] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 01:51:25,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 01:51:26,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:51:27,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:51:27,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:51:37,980] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank1 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0448 | Regression_eta: 0.0320 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0582 | LR: 1.54e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0035s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank1 [2026-10-30 02:01:47,468] INFO : Step 42200/50000 rank1 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0528 | Regression_eta: 0.0335 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0695 | LR: 1.53e-04 | DataLoad Time: 0.0162s | Model Forward Time: 7.2736s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank1 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank1 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-30 02:06:41,298] INFO : Step 42300/50000 rank1 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0488 | Regression_eta: 0.0246 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0540 | LR: 1.52e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8846s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank1 [2026-10-30 02:11:46,436] INFO : Step 42400/50000 rank1 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.0990 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0351 | Regression_eta: 0.0386 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0554 | LR: 1.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.1604s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank1 [2026-10-30 02:16:38,989] INFO : Step 42500/50000 rank1 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0406 | Regression_eta: 0.0292 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0557 | LR: 1.49e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8631s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank1 [2026-10-30 02:21:27,110] INFO : Step 42600/50000 rank1 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0436 | Regression_eta: 0.0348 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0553 | LR: 1.48e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.3577s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank1 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank1 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1056 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0423 | Regression_eta: 0.0247 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0638 | LR: 1.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6232s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank1 [2026-10-30 02:31:38,796] INFO : Step 42800/50000 rank1 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1260 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0408 | Regression_eta: 0.0414 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0492 | LR: 1.45e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6408s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank1 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank1 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0253 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0495 | Regression_eta: 0.0258 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0488 | LR: 1.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8537s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank1 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank1 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0377 | Regression_eta: 0.0285 | Regression_pt: 0.0386 | Regression_sin_phi: 0.0524 | LR: 1.43e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5960s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank1 [2026-10-30 02:41:14,878] INFO : Running validation on rank1 +rank1 [2026-10-30 02:41:14,878] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 02:41:14,878] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 02:42:10,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:42:10,782] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 02:42:38,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 02:42:39,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 02:42:40,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 02:42:41,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 02:42:51,235] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 02:42:51,953] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:42:52,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 02:43:08,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 02:43:09,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 02:43:10,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 02:43:11,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 02:43:21,225] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 02:43:38,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 02:43:39,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 02:43:39,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 02:43:40,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 02:43:50,811] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 02:48:51,376] INFO : Step 43100/50000 rank1 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0485 | Regression_eta: 0.0346 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0696 | LR: 1.42e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9465s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank1 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank1 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0559 | Regression_eta: 0.0514 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0638 | LR: 1.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5638s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank1 [2026-10-30 02:58:07,501] INFO : Step 43300/50000 rank1 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0348 | Regression_eta: 0.0323 | Regression_pt: 0.0353 | Regression_sin_phi: 0.0378 | LR: 1.39e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank1 [2026-10-30 03:02:42,807] INFO : Step 43400/50000 rank1 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1009 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0429 | Regression_eta: 0.0258 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0630 | LR: 1.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4740s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank1 [2026-10-30 03:07:16,794] INFO : Step 43500/50000 rank1 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1065 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0523 | Regression_eta: 0.0369 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0448 | LR: 1.37e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9338s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank1 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank1 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1833 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0412 | Regression_eta: 0.0432 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0576 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8412s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank1 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank1 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1235 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0571 | Regression_eta: 0.0316 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0486 | LR: 1.35e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1359s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank1 [2026-10-30 03:21:10,515] INFO : Step 43800/50000 rank1 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0451 | Regression_eta: 0.0244 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0300 | LR: 1.34e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2539s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank1 [2026-10-30 03:26:01,000] INFO : Step 43900/50000 rank1 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0381 | Regression_eta: 0.0469 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0392 | LR: 1.33e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9685s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank1 [2026-10-30 03:30:36,933] INFO : Step 44000/50000 rank1 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0531 | Regression_eta: 0.0337 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0559 | LR: 1.32e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8167s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank1 [2026-10-30 03:30:37,937] INFO : Running validation on rank1 +rank1 [2026-10-30 03:30:37,937] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 03:30:37,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 03:31:28,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:31:29,164] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 03:31:57,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 03:31:58,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 03:31:59,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 03:32:00,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 03:32:10,664] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 03:32:11,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 03:32:27,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 03:32:28,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 03:32:29,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 03:32:30,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 03:32:40,885] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:32:41,626] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 03:32:56,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 03:32:57,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 03:32:58,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 03:32:59,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 03:33:09,304] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 03:38:15,294] INFO : Step 44100/50000 rank1 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.0932 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0383 | Regression_eta: 0.0232 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0511 | LR: 1.31e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7479s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank1 [2026-10-30 03:43:04,522] INFO : Step 44200/50000 rank1 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0533 | Regression_eta: 0.0362 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0320 | LR: 1.30e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4224s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank1 [2026-10-30 03:47:46,465] INFO : Step 44300/50000 rank1 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0405 | Regression_eta: 0.0249 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0475 | LR: 1.29e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4338s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank1 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank1 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.0972 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0509 | Regression_eta: 0.0264 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0505 | LR: 1.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3354s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank1 [2026-10-30 03:57:11,971] INFO : Step 44500/50000 rank1 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0370 | Regression_eta: 0.0310 | Regression_pt: 0.0378 | Regression_sin_phi: 0.0513 | LR: 1.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3158s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank1 [2026-10-30 04:01:48,196] INFO : Step 44600/50000 rank1 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0224 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0411 | Regression_eta: 0.0228 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0219 | LR: 1.26e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5948s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank1 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank1 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.0986 | Regression_cos_phi: 0.0257 | Regression_energy: 0.0407 | Regression_eta: 0.0224 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0500 | LR: 1.25e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6239s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank1 [2026-10-30 04:11:07,220] INFO : Step 44800/50000 rank1 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0248 | Classification_binary: 0.1095 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0401 | Regression_eta: 0.0271 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0404 | LR: 1.24e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5863s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank1 [2026-10-30 04:15:54,537] INFO : Step 44900/50000 rank1 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0508 | Regression_eta: 0.0313 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0453 | LR: 1.23e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6734s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank1 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank1 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0539 | Regression_eta: 0.0478 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0618 | LR: 1.22e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7901s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank1 [2026-10-30 04:20:37,108] INFO : Running validation on rank1 +rank1 [2026-10-30 04:20:37,108] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 04:20:37,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 04:21:27,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:21:28,558] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 04:21:56,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 04:21:57,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 04:21:58,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 04:21:59,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 04:22:09,694] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:22:10,892] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 04:22:26,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 04:22:27,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 04:22:28,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 04:22:29,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 04:22:39,534] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:22:41,054] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 04:22:56,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 04:22:57,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 04:22:58,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 04:22:59,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 04:23:09,396] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank1 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0536 | Regression_eta: 0.0338 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0652 | LR: 1.21e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2108s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank1 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank1 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0253 | Regression_energy: 0.0436 | Regression_eta: 0.0246 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0443 | LR: 1.20e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4469s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank1 [2026-10-30 04:37:52,084] INFO : Step 45300/50000 rank1 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0330 | Regression_energy: 0.0412 | Regression_eta: 0.0304 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0459 | LR: 1.19e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8144s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank1 [2026-10-30 04:42:38,426] INFO : Step 45400/50000 rank1 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0658 | Regression_eta: 0.0331 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0375 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4020s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank1 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank1 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1154 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0427 | Regression_eta: 0.0200 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0403 | LR: 1.18e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3861s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank1 [2026-10-30 04:52:16,190] INFO : Step 45600/50000 rank1 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0487 | Regression_eta: 0.0538 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0573 | LR: 1.17e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4798s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank1 [2026-10-30 04:57:07,845] INFO : Step 45700/50000 rank1 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0419 | Regression_eta: 0.0364 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0503 | LR: 1.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4893s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank1 [2026-10-30 05:01:56,082] INFO : Step 45800/50000 rank1 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.0988 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0527 | Regression_eta: 0.0253 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0436 | LR: 1.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6463s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank1 [2026-10-30 05:06:42,062] INFO : Step 45900/50000 rank1 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1263 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0400 | Regression_eta: 0.0268 | Regression_pt: 0.0413 | Regression_sin_phi: 0.0350 | LR: 1.15e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2866s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank1 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank1 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0439 | Regression_eta: 0.0246 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0475 | LR: 1.14e-04 | DataLoad Time: 0.0224s | Model Forward Time: 6.4328s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank1 [2026-10-30 05:11:37,037] INFO : Running validation on rank1 +rank1 [2026-10-30 05:11:37,037] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 05:11:37,037] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 05:12:27,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:12:27,994] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 05:12:52,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 05:12:53,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 05:12:54,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 05:12:55,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 05:13:05,923] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 05:13:06,686] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 05:13:22,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 05:13:23,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 05:13:24,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 05:13:25,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 05:13:35,591] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 05:13:36,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 05:13:52,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 05:13:53,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 05:13:54,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 05:13:55,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 05:14:05,250] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 05:19:00,537] INFO : Step 46100/50000 rank1 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1018 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0446 | Regression_eta: 0.0253 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0589 | LR: 1.13e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5898s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank1 [2026-10-30 05:23:34,445] INFO : Step 46200/50000 rank1 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0527 | Regression_eta: 0.0286 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0452 | LR: 1.13e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7926s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank1 [2026-10-30 05:28:08,608] INFO : Step 46300/50000 rank1 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0367 | Regression_eta: 0.0429 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0493 | LR: 1.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9767s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank1 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank1 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0495 | Regression_eta: 0.0284 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0473 | LR: 1.11e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank1 [2026-10-30 05:37:21,831] INFO : Step 46500/50000 rank1 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1149 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0333 | Regression_eta: 0.0297 | Regression_pt: 0.0340 | Regression_sin_phi: 0.0500 | LR: 1.11e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8422s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank1 [2026-10-30 05:41:57,344] INFO : Step 46600/50000 rank1 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0541 | Regression_eta: 0.0341 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0586 | LR: 1.10e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7854s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank1 [2026-10-30 05:46:32,412] INFO : Step 46700/50000 rank1 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0590 | Regression_eta: 0.0400 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0429 | LR: 1.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6554s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank1 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank1 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.0973 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0438 | Regression_eta: 0.0254 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0319 | LR: 1.09e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7015s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank1 [2026-10-30 05:55:41,597] INFO : Step 46900/50000 rank1 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0432 | Regression_eta: 0.0255 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0452 | LR: 1.09e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7513s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank1 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank1 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0205 | Classification_binary: 0.1286 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0368 | Regression_eta: 0.0243 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0457 | LR: 1.08e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8059s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank1 [2026-10-30 06:00:21,749] INFO : Running validation on rank1 +rank1 [2026-10-30 06:00:21,749] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 06:01:12,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 06:01:37,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 06:01:38,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 06:01:39,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 06:01:40,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 06:01:50,292] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:01:51,182] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:01:51,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 06:02:07,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 06:02:08,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 06:02:09,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 06:02:10,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 06:02:20,493] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:02:21,870] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 06:02:37,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 06:02:38,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 06:02:38,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 06:02:39,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 06:02:49,769] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 06:07:51,194] INFO : Step 47100/50000 rank1 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0478 | Regression_eta: 0.0335 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0533 | LR: 1.07e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6257s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank1 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank1 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0261 | Classification_binary: 0.1152 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0362 | Regression_eta: 0.0317 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0493 | LR: 1.07e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9890s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank1 [2026-10-30 06:17:04,150] INFO : Step 47300/50000 rank1 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0347 | Regression_eta: 0.0278 | Regression_pt: 0.0355 | Regression_sin_phi: 0.0566 | LR: 1.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0313s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank1 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank1 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1048 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0555 | Regression_eta: 0.0259 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0328 | LR: 1.06e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4532s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank1 [2026-10-30 06:26:15,801] INFO : Step 47500/50000 rank1 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.0954 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0532 | Regression_eta: 0.0227 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0431 | LR: 1.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6770s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank1 [2026-10-30 06:30:50,118] INFO : Step 47600/50000 rank1 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0425 | Regression_eta: 0.0255 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0553 | LR: 1.05e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9483s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank1 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank1 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0448 | Regression_eta: 0.0362 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0634 | LR: 1.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9105s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank1 [2026-10-30 06:40:00,408] INFO : Step 47800/50000 rank1 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0391 | Regression_eta: 0.0371 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0386 | LR: 1.04e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7803s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank1 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank1 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0498 | Regression_eta: 0.0249 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0509 | LR: 1.04e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8487s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank1 [2026-10-30 06:49:09,637] INFO : Step 48000/50000 rank1 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0343 | Regression_eta: 0.0240 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0402 | LR: 1.04e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.8841s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank1 [2026-10-30 06:49:10,726] INFO : Running validation on rank1 +rank1 [2026-10-30 06:49:10,726] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 06:49:10,726] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 06:50:01,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 06:50:27,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 06:50:28,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 06:50:29,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 06:50:30,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 06:50:40,308] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:50:41,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:50:41,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 06:50:57,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 06:50:58,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 06:50:59,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 06:51:00,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 06:51:10,756] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:51:10,927] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:51:11,435] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 06:51:26,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 06:51:27,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 06:51:28,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 06:51:29,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 06:51:39,029] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 06:56:35,098] INFO : Step 48100/50000 rank1 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0426 | Regression_eta: 0.0302 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0374 | LR: 1.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6290s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank1 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank1 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.0985 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0438 | Regression_eta: 0.0322 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0445 | LR: 1.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3157s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank1 [2026-10-30 07:05:53,465] INFO : Step 48300/50000 rank1 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0414 | Regression_eta: 0.0292 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0510 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1170s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank1 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank1 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0399 | Regression_eta: 0.0373 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0385 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8695s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank1 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank1 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1002 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0445 | Regression_eta: 0.0336 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0488 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.9625s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank1 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank1 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0340 | Regression_eta: 0.0297 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0452 | LR: 1.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8067s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank1 [2026-10-30 07:24:30,057] INFO : Step 48700/50000 rank1 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0304 | Regression_energy: 0.0322 | Regression_eta: 0.0215 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0411 | LR: 1.02e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9968s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank1 [2026-10-30 07:29:10,218] INFO : Step 48800/50000 rank1 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1190 | Regression_cos_phi: 0.0337 | Regression_energy: 0.0388 | Regression_eta: 0.0223 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0471 | LR: 1.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9236s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank1 [2026-10-30 07:33:48,296] INFO : Step 48900/50000 rank1 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1081 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0411 | Regression_eta: 0.0300 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0443 | LR: 1.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7881s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank1 [2026-10-30 07:38:22,688] INFO : Step 49000/50000 rank1 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0447 | Regression_eta: 0.0268 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0385 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8889s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank1 [2026-10-30 07:38:23,698] INFO : Running validation on rank1 +rank1 [2026-10-30 07:38:23,698] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 07:38:23,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 07:39:14,060] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:39:14,628] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 07:39:39,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 07:39:40,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 07:39:41,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 07:39:42,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 07:39:52,303] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 07:39:53,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:39:53,776] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 07:40:09,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 07:40:10,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 07:40:11,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 07:40:12,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 07:40:22,212] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:40:24,011] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 07:40:39,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 07:40:40,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 07:40:41,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 07:40:41,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 07:40:51,912] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank1 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1088 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0409 | Regression_eta: 0.0203 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0266 | LR: 1.01e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6568s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank1 [2026-10-30 07:50:30,114] INFO : Step 49200/50000 rank1 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.0987 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0386 | Regression_eta: 0.0186 | Regression_pt: 0.0386 | Regression_sin_phi: 0.0302 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5152s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank1 [2026-10-30 07:55:07,371] INFO : Step 49300/50000 rank1 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0253 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0358 | Regression_eta: 0.0286 | Regression_pt: 0.0382 | Regression_sin_phi: 0.0498 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8683s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank1 [2026-10-30 07:59:43,350] INFO : Step 49400/50000 rank1 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0395 | Regression_eta: 0.0506 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0497 | LR: 1.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8895s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank1 [2026-10-30 08:04:24,759] INFO : Step 49500/50000 rank1 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1144 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0368 | Regression_eta: 0.0219 | Regression_pt: 0.0383 | Regression_sin_phi: 0.0342 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3352s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank1 [2026-10-30 08:08:58,923] INFO : Step 49600/50000 rank1 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0346 | Regression_eta: 0.0232 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0578 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8762s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank1 [2026-10-30 08:13:36,932] INFO : Step 49700/50000 rank1 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0506 | Regression_eta: 0.0313 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0412 | LR: 1.00e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.7062s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank1 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank1 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0258 | Classification_binary: 0.0856 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0319 | Regression_eta: 0.0205 | Regression_pt: 0.0327 | Regression_sin_phi: 0.0452 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5269s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank1 [2026-10-30 08:22:53,416] INFO : Step 49900/50000 rank1 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1129 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0555 | Regression_eta: 0.0249 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0245 | LR: 1.00e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.2010s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank1 [2026-10-30 08:27:37,250] INFO : Step 50000/50000 rank1 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0440 | Regression_eta: 0.0250 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0486 | LR: 1.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3467s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank1 [2026-10-30 08:27:38,186] INFO : Running validation on rank1 +rank1 [2026-10-30 08:27:38,186] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 08:27:38,186] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 08:28:29,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 08:28:29,585] INFO : Running predictions on clic_edm_ttbar_hits +rank1 [2026-10-30 08:29:00,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 08:29:01,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 08:29:02,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 08:29:03,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 08:29:13,176] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 08:29:13,910] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 08:29:14,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank1 [2026-10-30 08:29:30,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 08:29:31,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 08:29:32,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 08:29:33,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 08:29:43,933] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 08:29:46,174] INFO : Running predictions on clic_edm_qq_hits +rank1 [2026-10-30 08:30:01,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 08:30:02,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 08:30:03,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 08:30:04,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 08:30:14,155] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 08:30:15,223] INFO : Training completed. Total time on device 1: 1059.923min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.2 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..9dfa9745c3e2f45165ca6bf5bb0799759932c9c5 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.2 @@ -0,0 +1,2654 @@ +rank2 [2026-10-28 11:12:32,405] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-28 11:12:32,406] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank2 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank2 [2026-10-28 11:12:57,209] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-28 11:12:57,209] INFO : Initializing backbone layer 0 +rank2 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank2 [2026-10-28 11:12:57,254] INFO : Initializing backbone layer 2 +rank2 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 3 +rank2 [2026-10-28 11:12:57,272] INFO : Initializing backbone layer 4 +rank2 [2026-10-28 11:12:57,280] INFO : Initializing backbone layer 5 +rank2 [2026-10-28 11:12:57,289] INFO : Backbone initialization took 0.08s +rank2 [2026-10-28 11:12:57,289] INFO : backbone parameters: 7901376 +rank2 [2026-10-28 11:12:57,290] INFO : Initializing output DNNs +rank2 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank2 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank2 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank2 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank2 [2026-10-28 11:12:57,329] INFO : classification_norm parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : regression_norm parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : classification_query parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : regression_query parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : classification_readout parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : regression_readout parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : output_mode=set +rank2 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank2 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank2 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank2 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank2 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=2 +rank2 [2026-10-28 11:12:57,370] INFO : Moved model to device rank=2 +rank2 [2026-10-28 11:12:57,372] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-28 11:12:57,372] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-28 11:12:57,857] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank2 [2026-10-28 11:12:57,857] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-28 11:12:57,858] INFO : Trainable parameters: 12786964 +rank2 [2026-10-28 11:12:57,858] INFO : Non-trainable parameters: 23268 +rank2 [2026-10-28 11:12:57,858] INFO : Total parameters: 12810232 +rank2 [2026-10-28 11:12:57,861] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-28 11:12:57,862] INFO : Getting dataloaders +rank2 [2026-10-28 11:12:57,862] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:03,726] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:04,753] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,002] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,274] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,533] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,794] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-28 11:13:06,794] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-28 11:13:20,899] INFO : Got dataloaders +rank2 [2026-10-28 11:13:20,899] INFO : loader split=train rank=2 len=21108 +rank2 [2026-10-28 11:13:20,899] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-28 11:13:20,900] INFO : Starting training from step 1 to 50000 on rank 2 of 8 +rank2 [2026-10-28 11:13:20,900] INFO : Creating train_iterator +rank2 [2026-10-28 11:13:20,900] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-28 11:13:35,609] INFO : Created train_iterator +rank2 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank2 [2026-10-28 11:18:16,510] INFO : Step 100/50000 rank2 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7785 | Classification_binary: 0.4935 | Regression_cos_phi: 0.0049 | Regression_energy: 1.0701 | Regression_eta: 0.0117 | Regression_pt: 1.0136 | Regression_sin_phi: 0.0067 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 3.6166s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank2 [2026-10-28 11:22:47,963] INFO : Step 200/50000 rank2 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 0.5405 | Regression_cos_phi: 0.3149 | Regression_energy: 0.3446 | Regression_eta: 0.3341 | Regression_pt: 0.3430 | Regression_sin_phi: 0.4392 | LR: 1.00e-03 | DataLoad Time: 0.0169s | Model Forward Time: 4.4383s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank2 [2026-10-28 11:27:17,891] INFO : Step 300/50000 rank2 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3103 | Classification_binary: 0.4545 | Regression_cos_phi: 0.3203 | Regression_energy: 0.3202 | Regression_eta: 0.3370 | Regression_pt: 0.3216 | Regression_sin_phi: 0.3706 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 4.3690s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank2 [2026-10-28 11:31:48,894] INFO : Step 400/50000 rank2 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2444 | Classification_binary: 0.4668 | Regression_cos_phi: 0.3879 | Regression_energy: 0.3489 | Regression_eta: 0.2668 | Regression_pt: 0.3609 | Regression_sin_phi: 0.3123 | LR: 1.00e-03 | DataLoad Time: 0.0169s | Model Forward Time: 4.2442s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank2 [2026-10-28 11:36:20,165] INFO : Step 500/50000 rank2 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2926 | Classification_binary: 0.4789 | Regression_cos_phi: 0.2985 | Regression_energy: 0.2632 | Regression_eta: 0.2480 | Regression_pt: 0.2722 | Regression_sin_phi: 0.3298 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.4689s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank2 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank2 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2421 | Classification_binary: 0.5093 | Regression_cos_phi: 0.2650 | Regression_energy: 0.2941 | Regression_eta: 0.2166 | Regression_pt: 0.3104 | Regression_sin_phi: 0.2477 | LR: 1.00e-03 | DataLoad Time: 0.0149s | Model Forward Time: 4.5151s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank2 [2026-10-28 11:45:32,573] INFO : Step 700/50000 rank2 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2141 | Classification_binary: 0.4163 | Regression_cos_phi: 0.2593 | Regression_energy: 0.2815 | Regression_eta: 0.1693 | Regression_pt: 0.2884 | Regression_sin_phi: 0.1778 | LR: 1.00e-03 | DataLoad Time: 0.0173s | Model Forward Time: 3.7405s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank2 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank2 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2573 | Classification_binary: 0.4307 | Regression_cos_phi: 0.3011 | Regression_energy: 0.3219 | Regression_eta: 0.3236 | Regression_pt: 0.3329 | Regression_sin_phi: 0.3001 | LR: 9.99e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.2804s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank2 [2026-10-28 11:54:58,536] INFO : Step 900/50000 rank2 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1880 | Classification_binary: 0.4819 | Regression_cos_phi: 0.2376 | Regression_energy: 0.2453 | Regression_eta: 0.1810 | Regression_pt: 0.2511 | Regression_sin_phi: 0.2135 | LR: 9.99e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4582s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank2 [2026-10-28 11:59:55,232] INFO : Step 1000/50000 rank2 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1654 | Classification_binary: 0.3919 | Regression_cos_phi: 0.2119 | Regression_energy: 0.2868 | Regression_eta: 0.1581 | Regression_pt: 0.2855 | Regression_sin_phi: 0.1550 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.7402s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank2 [2026-10-28 11:59:56,332] INFO : Running validation on rank2 +rank2 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 12:00:48,695] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:00:49,301] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 12:01:16,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:01:17,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:01:18,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:01:19,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:01:29,957] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:01:30,839] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 12:01:46,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:01:47,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:01:48,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:01:49,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:01:59,504] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:02:00,465] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:02:01,007] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 12:02:16,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:02:17,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:02:18,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:02:19,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:02:29,232] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 12:07:33,552] INFO : Step 1100/50000 rank2 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1932 | Classification_binary: 0.3972 | Regression_cos_phi: 0.1727 | Regression_energy: 0.2636 | Regression_eta: 0.1522 | Regression_pt: 0.2742 | Regression_sin_phi: 0.1918 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8135s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank2 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank2 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2205 | Classification_binary: 0.4479 | Regression_cos_phi: 0.2635 | Regression_energy: 0.2459 | Regression_eta: 0.1695 | Regression_pt: 0.2508 | Regression_sin_phi: 0.2488 | LR: 9.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4363s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank2 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank2 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2064 | Classification_binary: 0.4311 | Regression_cos_phi: 0.2076 | Regression_energy: 0.2323 | Regression_eta: 0.1844 | Regression_pt: 0.2412 | Regression_sin_phi: 0.2662 | LR: 9.98e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2255s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank2 [2026-10-28 12:21:55,310] INFO : Step 1400/50000 rank2 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1735 | Classification_binary: 0.3383 | Regression_cos_phi: 0.2020 | Regression_energy: 0.2769 | Regression_eta: 0.1827 | Regression_pt: 0.2847 | Regression_sin_phi: 0.2728 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5550s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank2 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank2 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1947 | Classification_binary: 0.4393 | Regression_cos_phi: 0.2390 | Regression_energy: 0.2574 | Regression_eta: 0.1714 | Regression_pt: 0.2676 | Regression_sin_phi: 0.2752 | LR: 9.98e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9968s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank2 [2026-10-28 12:31:13,938] INFO : Step 1600/50000 rank2 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1626 | Classification_binary: 0.3831 | Regression_cos_phi: 0.1909 | Regression_energy: 0.1887 | Regression_eta: 0.1277 | Regression_pt: 0.1956 | Regression_sin_phi: 0.1747 | LR: 9.98e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8063s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank2 [2026-10-28 12:36:04,098] INFO : Step 1700/50000 rank2 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1570 | Classification_binary: 0.3549 | Regression_cos_phi: 0.2048 | Regression_energy: 0.2112 | Regression_eta: 0.1742 | Regression_pt: 0.2184 | Regression_sin_phi: 0.2094 | LR: 9.97e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6769s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank2 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank2 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1858 | Classification_binary: 0.3799 | Regression_cos_phi: 0.1808 | Regression_energy: 0.2330 | Regression_eta: 0.1448 | Regression_pt: 0.2385 | Regression_sin_phi: 0.2037 | LR: 9.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6773s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank2 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank2 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1764 | Classification_binary: 0.4133 | Regression_cos_phi: 0.2572 | Regression_energy: 0.2516 | Regression_eta: 0.1700 | Regression_pt: 0.2592 | Regression_sin_phi: 0.2081 | LR: 9.97e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4306s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank2 [2026-10-28 12:50:22,600] INFO : Step 2000/50000 rank2 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1523 | Classification_binary: 0.3597 | Regression_cos_phi: 0.2101 | Regression_energy: 0.2315 | Regression_eta: 0.1537 | Regression_pt: 0.2384 | Regression_sin_phi: 0.2251 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9905s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank2 [2026-10-28 12:50:23,689] INFO : Running validation on rank2 +rank2 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 12:51:19,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:51:20,337] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 12:51:47,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:51:48,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:51:50,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:51:51,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:52:01,277] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 12:52:18,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:52:19,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:52:20,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:52:21,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:52:31,601] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:52:31,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:52:32,091] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 12:52:48,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:52:49,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:52:50,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:52:50,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:53:00,996] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank2 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1273 | Classification_binary: 0.3071 | Regression_cos_phi: 0.1315 | Regression_energy: 0.1926 | Regression_eta: 0.0753 | Regression_pt: 0.1979 | Regression_sin_phi: 0.1156 | LR: 9.96e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6446s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank2 [2026-10-28 13:03:25,046] INFO : Step 2200/50000 rank2 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2028 | Classification_binary: 0.3872 | Regression_cos_phi: 0.2323 | Regression_energy: 0.2206 | Regression_eta: 0.1818 | Regression_pt: 0.2326 | Regression_sin_phi: 0.2292 | LR: 9.96e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.4333s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank2 [2026-10-28 13:08:12,323] INFO : Step 2300/50000 rank2 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1854 | Classification_binary: 0.4408 | Regression_cos_phi: 0.2389 | Regression_energy: 0.2423 | Regression_eta: 0.1910 | Regression_pt: 0.2508 | Regression_sin_phi: 0.1800 | LR: 9.95e-04 | DataLoad Time: 0.0137s | Model Forward Time: 5.1036s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank2 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank2 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1713 | Classification_binary: 0.2958 | Regression_cos_phi: 0.1850 | Regression_energy: 0.2027 | Regression_eta: 0.1126 | Regression_pt: 0.2074 | Regression_sin_phi: 0.1355 | LR: 9.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2192s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank2 [2026-10-28 13:18:06,886] INFO : Step 2500/50000 rank2 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1961 | Classification_binary: 0.4052 | Regression_cos_phi: 0.2206 | Regression_energy: 0.2114 | Regression_eta: 0.1787 | Regression_pt: 0.2174 | Regression_sin_phi: 0.1958 | LR: 9.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.0092s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank2 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank2 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1969 | Classification_binary: 0.4749 | Regression_cos_phi: 0.2099 | Regression_energy: 0.2314 | Regression_eta: 0.1453 | Regression_pt: 0.2407 | Regression_sin_phi: 0.2126 | LR: 9.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3777s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank2 [2026-10-28 13:27:39,806] INFO : Step 2700/50000 rank2 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1509 | Classification_binary: 0.3241 | Regression_cos_phi: 0.1916 | Regression_energy: 0.1967 | Regression_eta: 0.1214 | Regression_pt: 0.2029 | Regression_sin_phi: 0.1568 | LR: 9.94e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5221s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank2 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank2 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2058 | Classification_binary: 0.3493 | Regression_cos_phi: 0.2197 | Regression_energy: 0.2380 | Regression_eta: 0.1556 | Regression_pt: 0.2470 | Regression_sin_phi: 0.2628 | LR: 9.93e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2919s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank2 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank2 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2030 | Classification_binary: 0.4431 | Regression_cos_phi: 0.2804 | Regression_energy: 0.2146 | Regression_eta: 0.1974 | Regression_pt: 0.2221 | Regression_sin_phi: 0.2524 | LR: 9.93e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5107s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank2 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank2 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1491 | Classification_binary: 0.3810 | Regression_cos_phi: 0.1493 | Regression_energy: 0.1773 | Regression_eta: 0.1363 | Regression_pt: 0.1861 | Regression_sin_phi: 0.1807 | LR: 9.92e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank2 [2026-10-28 13:42:04,672] INFO : Running validation on rank2 +rank2 [2026-10-28 13:42:04,673] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:42:04,673] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:42:57,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:42:58,470] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 13:43:25,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:43:26,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:43:27,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:43:28,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:43:38,290] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:43:38,829] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:43:39,344] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 13:43:55,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:43:56,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:43:57,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:43:58,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:44:08,534] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:44:09,100] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 13:44:24,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:44:25,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:44:26,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:44:27,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:44:37,300] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 13:49:29,793] INFO : Step 3100/50000 rank2 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1563 | Classification_binary: 0.2989 | Regression_cos_phi: 0.2037 | Regression_energy: 0.2004 | Regression_eta: 0.1342 | Regression_pt: 0.2094 | Regression_sin_phi: 0.1532 | LR: 9.91e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8443s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank2 [2026-10-28 13:54:22,657] INFO : Step 3200/50000 rank2 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1789 | Classification_binary: 0.3920 | Regression_cos_phi: 0.2363 | Regression_energy: 0.2170 | Regression_eta: 0.1360 | Regression_pt: 0.2257 | Regression_sin_phi: 0.2280 | LR: 9.91e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.0524s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank2 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank2 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1544 | Classification_binary: 0.3839 | Regression_cos_phi: 0.1802 | Regression_energy: 0.1987 | Regression_eta: 0.1439 | Regression_pt: 0.2089 | Regression_sin_phi: 0.2025 | LR: 9.90e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7404s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank2 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank2 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1184 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1136 | Regression_energy: 0.2185 | Regression_eta: 0.1185 | Regression_pt: 0.2227 | Regression_sin_phi: 0.1241 | LR: 9.90e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9447s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank2 [2026-10-28 14:08:16,394] INFO : Step 3500/50000 rank2 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1396 | Classification_binary: 0.3478 | Regression_cos_phi: 0.1935 | Regression_energy: 0.1945 | Regression_eta: 0.1259 | Regression_pt: 0.2012 | Regression_sin_phi: 0.1917 | LR: 9.89e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7441s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank2 [2026-10-28 14:13:02,152] INFO : Step 3600/50000 rank2 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1580 | Classification_binary: 0.4249 | Regression_cos_phi: 0.2675 | Regression_energy: 0.1989 | Regression_eta: 0.1582 | Regression_pt: 0.2064 | Regression_sin_phi: 0.2381 | LR: 9.89e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8790s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank2 [2026-10-28 14:17:42,402] INFO : Step 3700/50000 rank2 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1513 | Classification_binary: 0.3600 | Regression_cos_phi: 0.2107 | Regression_energy: 0.1747 | Regression_eta: 0.1474 | Regression_pt: 0.1789 | Regression_sin_phi: 0.1781 | LR: 9.88e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2549s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank2 [2026-10-28 14:22:44,343] INFO : Step 3800/50000 rank2 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1242 | Classification_binary: 0.3335 | Regression_cos_phi: 0.1541 | Regression_energy: 0.2043 | Regression_eta: 0.1567 | Regression_pt: 0.2141 | Regression_sin_phi: 0.1730 | LR: 9.87e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1812s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank2 [2026-10-28 14:27:33,631] INFO : Step 3900/50000 rank2 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1414 | Classification_binary: 0.3875 | Regression_cos_phi: 0.2157 | Regression_energy: 0.1983 | Regression_eta: 0.1250 | Regression_pt: 0.2037 | Regression_sin_phi: 0.2099 | LR: 9.87e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.8160s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank2 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank2 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1291 | Classification_binary: 0.3571 | Regression_cos_phi: 0.1904 | Regression_energy: 0.1712 | Regression_eta: 0.1180 | Regression_pt: 0.1782 | Regression_sin_phi: 0.1847 | LR: 9.86e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5853s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank2 [2026-10-28 14:32:39,608] INFO : Running validation on rank2 +rank2 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 14:33:34,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:33:34,720] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 14:34:03,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 14:34:05,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 14:34:10,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 14:34:11,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 14:34:21,645] INFO : Time taken to make predictions on device 2 is: 0.35 min +rank2 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:34:24,281] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 14:34:40,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 14:34:43,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 14:34:44,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 14:34:45,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 14:34:55,782] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:34:56,345] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 14:35:11,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 14:35:12,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 14:35:13,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 14:35:14,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 14:35:24,192] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank2 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1673 | Regression_energy: 0.1915 | Regression_eta: 0.1278 | Regression_pt: 0.1943 | Regression_sin_phi: 0.2552 | LR: 9.85e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8518s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank2 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank2 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1733 | Classification_binary: 0.3517 | Regression_cos_phi: 0.2141 | Regression_energy: 0.2172 | Regression_eta: 0.1412 | Regression_pt: 0.2239 | Regression_sin_phi: 0.2211 | LR: 9.84e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5072s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank2 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank2 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1489 | Classification_binary: 0.4042 | Regression_cos_phi: 0.1473 | Regression_energy: 0.1858 | Regression_eta: 0.1430 | Regression_pt: 0.1951 | Regression_sin_phi: 0.2168 | LR: 9.84e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4590s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank2 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank2 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1126 | Classification_binary: 0.3130 | Regression_cos_phi: 0.1670 | Regression_energy: 0.1916 | Regression_eta: 0.1127 | Regression_pt: 0.1986 | Regression_sin_phi: 0.1939 | LR: 9.83e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1738s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank2 [2026-10-28 14:59:28,686] INFO : Step 4500/50000 rank2 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1214 | Classification_binary: 0.3146 | Regression_cos_phi: 0.1361 | Regression_energy: 0.2068 | Regression_eta: 0.0974 | Regression_pt: 0.2138 | Regression_sin_phi: 0.1484 | LR: 9.82e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2506s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank2 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank2 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1413 | Classification_binary: 0.3607 | Regression_cos_phi: 0.1921 | Regression_energy: 0.1914 | Regression_eta: 0.0978 | Regression_pt: 0.1984 | Regression_sin_phi: 0.1790 | LR: 9.81e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7023s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank2 [2026-10-28 15:09:18,561] INFO : Step 4700/50000 rank2 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.2804 | Regression_cos_phi: 0.1307 | Regression_energy: 0.1890 | Regression_eta: 0.1182 | Regression_pt: 0.1953 | Regression_sin_phi: 0.1656 | LR: 9.81e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7057s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank2 [2026-10-28 15:14:30,754] INFO : Step 4800/50000 rank2 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 0.2881 | Regression_cos_phi: 0.2055 | Regression_energy: 0.1872 | Regression_eta: 0.1278 | Regression_pt: 0.1981 | Regression_sin_phi: 0.1768 | LR: 9.80e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4405s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank2 [2026-10-28 15:19:27,570] INFO : Step 4900/50000 rank2 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1421 | Classification_binary: 0.3579 | Regression_cos_phi: 0.1920 | Regression_energy: 0.1930 | Regression_eta: 0.1213 | Regression_pt: 0.2026 | Regression_sin_phi: 0.1716 | LR: 9.79e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5347s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank2 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank2 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 0.3414 | Regression_cos_phi: 0.1917 | Regression_energy: 0.1861 | Regression_eta: 0.1397 | Regression_pt: 0.1886 | Regression_sin_phi: 0.2025 | LR: 9.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4820s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank2 [2026-10-28 15:24:24,982] INFO : Running validation on rank2 +rank2 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 15:25:19,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:25:19,574] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 15:25:44,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 15:25:45,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 15:25:47,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 15:25:48,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 15:25:58,069] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:25:59,251] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 15:26:15,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 15:26:16,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 15:26:17,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 15:26:18,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 15:26:28,276] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:26:28,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 15:26:44,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 15:26:45,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 15:26:46,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 15:26:47,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 15:26:57,458] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 15:32:01,930] INFO : Step 5100/50000 rank2 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.2759 | Regression_cos_phi: 0.1560 | Regression_energy: 0.1773 | Regression_eta: 0.1287 | Regression_pt: 0.1813 | Regression_sin_phi: 0.1532 | LR: 9.77e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.6406s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank2 [2026-10-28 15:36:53,514] INFO : Step 5200/50000 rank2 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 0.3205 | Regression_cos_phi: 0.1775 | Regression_energy: 0.1835 | Regression_eta: 0.1209 | Regression_pt: 0.1908 | Regression_sin_phi: 0.1643 | LR: 9.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6377s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank2 [2026-10-28 15:41:36,525] INFO : Step 5300/50000 rank2 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1162 | Classification_binary: 0.3588 | Regression_cos_phi: 0.1646 | Regression_energy: 0.1611 | Regression_eta: 0.1233 | Regression_pt: 0.1662 | Regression_sin_phi: 0.1782 | LR: 9.75e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9238s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank2 [2026-10-28 15:46:33,435] INFO : Step 5400/50000 rank2 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1278 | Classification_binary: 0.3267 | Regression_cos_phi: 0.1381 | Regression_energy: 0.1438 | Regression_eta: 0.1296 | Regression_pt: 0.1514 | Regression_sin_phi: 0.1762 | LR: 9.74e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4641s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank2 [2026-10-28 15:51:26,878] INFO : Step 5500/50000 rank2 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1440 | Classification_binary: 0.3159 | Regression_cos_phi: 0.1985 | Regression_energy: 0.1601 | Regression_eta: 0.1141 | Regression_pt: 0.1661 | Regression_sin_phi: 0.1762 | LR: 9.73e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.5346s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank2 [2026-10-28 15:56:25,711] INFO : Step 5600/50000 rank2 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1454 | Classification_binary: 0.3651 | Regression_cos_phi: 0.2164 | Regression_energy: 0.1791 | Regression_eta: 0.1295 | Regression_pt: 0.1875 | Regression_sin_phi: 0.1777 | LR: 9.72e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.3202s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank2 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank2 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1135 | Classification_binary: 0.2961 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1487 | Regression_eta: 0.0831 | Regression_pt: 0.1543 | Regression_sin_phi: 0.1109 | LR: 9.71e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.5507s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank2 [2026-10-28 16:06:09,459] INFO : Step 5800/50000 rank2 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1515 | Classification_binary: 0.2953 | Regression_cos_phi: 0.2104 | Regression_energy: 0.1869 | Regression_eta: 0.1550 | Regression_pt: 0.1943 | Regression_sin_phi: 0.1917 | LR: 9.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2360s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank2 [2026-10-28 16:11:19,901] INFO : Step 5900/50000 rank2 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1415 | Classification_binary: 0.3482 | Regression_cos_phi: 0.2009 | Regression_energy: 0.1921 | Regression_eta: 0.1446 | Regression_pt: 0.1989 | Regression_sin_phi: 0.2181 | LR: 9.69e-04 | DataLoad Time: 0.0164s | Model Forward Time: 7.1474s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank2 [2026-10-28 16:16:40,507] INFO : Step 6000/50000 rank2 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1178 | Classification_binary: 0.3321 | Regression_cos_phi: 0.1629 | Regression_energy: 0.1590 | Regression_eta: 0.1191 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1664 | LR: 9.68e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4949s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank2 [2026-10-28 16:16:41,624] INFO : Running validation on rank2 +rank2 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:17:32,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:17:33,086] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 16:17:59,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:18:00,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:18:01,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:18:02,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:18:12,526] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:18:13,324] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:18:13,876] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 16:18:29,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:18:31,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:18:32,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:18:32,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:18:42,989] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:18:43,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:18:44,188] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 16:18:59,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:19:01,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:19:02,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:19:03,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:19:13,896] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank2 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.3205 | Regression_cos_phi: 0.1712 | Regression_energy: 0.1596 | Regression_eta: 0.1439 | Regression_pt: 0.1684 | Regression_sin_phi: 0.2078 | LR: 9.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9564s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank2 [2026-10-28 16:29:26,888] INFO : Step 6200/50000 rank2 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1206 | Classification_binary: 0.2604 | Regression_cos_phi: 0.1513 | Regression_energy: 0.1649 | Regression_eta: 0.1155 | Regression_pt: 0.1713 | Regression_sin_phi: 0.1812 | LR: 9.66e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0889s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank2 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank2 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.3441 | Regression_cos_phi: 0.1562 | Regression_energy: 0.1653 | Regression_eta: 0.0945 | Regression_pt: 0.1719 | Regression_sin_phi: 0.1359 | LR: 9.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0309s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank2 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank2 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1052 | Classification_binary: 0.2848 | Regression_cos_phi: 0.1513 | Regression_energy: 0.1561 | Regression_eta: 0.0907 | Regression_pt: 0.1663 | Regression_sin_phi: 0.1248 | LR: 9.64e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7766s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank2 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank2 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2682 | Regression_cos_phi: 0.1346 | Regression_energy: 0.1630 | Regression_eta: 0.1010 | Regression_pt: 0.1760 | Regression_sin_phi: 0.1779 | LR: 9.63e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.4997s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank2 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank2 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.3346 | Regression_cos_phi: 0.1473 | Regression_energy: 0.1625 | Regression_eta: 0.1118 | Regression_pt: 0.1699 | Regression_sin_phi: 0.1580 | LR: 9.62e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8065s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank2 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank2 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.3159 | Regression_cos_phi: 0.1278 | Regression_energy: 0.1703 | Regression_eta: 0.1020 | Regression_pt: 0.1749 | Regression_sin_phi: 0.1509 | LR: 9.61e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.3363s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank2 [2026-10-28 16:58:34,482] INFO : Step 6800/50000 rank2 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1146 | Classification_binary: 0.2440 | Regression_cos_phi: 0.1410 | Regression_energy: 0.1662 | Regression_eta: 0.0955 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1650 | LR: 9.60e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.4640s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank2 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank2 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1363 | Classification_binary: 0.3310 | Regression_cos_phi: 0.1834 | Regression_energy: 0.1439 | Regression_eta: 0.1143 | Regression_pt: 0.1490 | Regression_sin_phi: 0.1839 | LR: 9.58e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6887s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank2 [2026-10-28 17:08:29,914] INFO : Step 7000/50000 rank2 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1430 | Classification_binary: 0.3346 | Regression_cos_phi: 0.1766 | Regression_energy: 0.1706 | Regression_eta: 0.1213 | Regression_pt: 0.1754 | Regression_sin_phi: 0.1780 | LR: 9.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9221s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank2 [2026-10-28 17:08:30,792] INFO : Running validation on rank2 +rank2 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 17:09:23,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:09:24,267] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 17:09:52,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 17:09:53,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 17:09:55,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 17:09:56,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 17:10:06,904] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-28 17:10:07,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:10:08,263] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 17:10:23,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 17:10:24,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 17:10:25,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 17:10:26,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 17:10:36,855] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:10:37,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:10:38,448] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 17:10:54,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 17:10:55,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 17:10:55,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 17:10:56,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 17:11:06,631] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 17:16:27,594] INFO : Step 7100/50000 rank2 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1058 | Classification_binary: 0.2774 | Regression_cos_phi: 0.1434 | Regression_energy: 0.1517 | Regression_eta: 0.1333 | Regression_pt: 0.1562 | Regression_sin_phi: 0.1875 | LR: 9.56e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1470s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank2 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank2 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1181 | Classification_binary: 0.2368 | Regression_cos_phi: 0.1253 | Regression_energy: 0.1376 | Regression_eta: 0.0825 | Regression_pt: 0.1407 | Regression_sin_phi: 0.1141 | LR: 9.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3456s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank2 [2026-10-28 17:26:34,943] INFO : Step 7300/50000 rank2 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1393 | Classification_binary: 0.3906 | Regression_cos_phi: 0.1661 | Regression_energy: 0.1377 | Regression_eta: 0.1131 | Regression_pt: 0.1412 | Regression_sin_phi: 0.1603 | LR: 9.53e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4029s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank2 [2026-10-28 17:31:34,684] INFO : Step 7400/50000 rank2 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.3053 | Regression_cos_phi: 0.1308 | Regression_energy: 0.1795 | Regression_eta: 0.0965 | Regression_pt: 0.1884 | Regression_sin_phi: 0.1112 | LR: 9.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5948s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank2 [2026-10-28 17:36:30,885] INFO : Step 7500/50000 rank2 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.2535 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1475 | Regression_eta: 0.1015 | Regression_pt: 0.1533 | Regression_sin_phi: 0.1241 | LR: 9.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6605s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank2 [2026-10-28 17:41:09,113] INFO : Step 7600/50000 rank2 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1356 | Classification_binary: 0.3354 | Regression_cos_phi: 0.1925 | Regression_energy: 0.1517 | Regression_eta: 0.1413 | Regression_pt: 0.1569 | Regression_sin_phi: 0.1804 | LR: 9.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7910s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank2 [2026-10-28 17:46:03,243] INFO : Step 7700/50000 rank2 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1182 | Classification_binary: 0.3438 | Regression_cos_phi: 0.1472 | Regression_energy: 0.1634 | Regression_eta: 0.1045 | Regression_pt: 0.1708 | Regression_sin_phi: 0.1686 | LR: 9.48e-04 | DataLoad Time: 0.0125s | Model Forward Time: 4.7867s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank2 [2026-10-28 17:51:02,467] INFO : Step 7800/50000 rank2 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.2674 | Regression_cos_phi: 0.1239 | Regression_energy: 0.1245 | Regression_eta: 0.1014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.1407 | LR: 9.47e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4033s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank2 [2026-10-28 17:56:05,210] INFO : Step 7900/50000 rank2 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1183 | Classification_binary: 0.3214 | Regression_cos_phi: 0.1601 | Regression_energy: 0.1589 | Regression_eta: 0.0964 | Regression_pt: 0.1662 | Regression_sin_phi: 0.1352 | LR: 9.46e-04 | DataLoad Time: 0.0218s | Model Forward Time: 6.3962s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank2 [2026-10-28 18:00:55,277] INFO : Step 8000/50000 rank2 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.3508 | Regression_cos_phi: 0.1739 | Regression_energy: 0.1484 | Regression_eta: 0.1355 | Regression_pt: 0.1550 | Regression_sin_phi: 0.1729 | LR: 9.44e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4819s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank2 [2026-10-28 18:00:56,228] INFO : Running validation on rank2 +rank2 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:01:47,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:01:48,313] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 18:02:14,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:02:15,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:02:16,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:02:17,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:02:28,008] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:02:28,859] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 18:02:45,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:02:46,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:02:47,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:02:48,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:02:58,062] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:02:58,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:02:58,745] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 18:03:14,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:03:15,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:03:16,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:03:17,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:03:27,085] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 18:08:30,021] INFO : Step 8100/50000 rank2 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0945 | Classification_binary: 0.2508 | Regression_cos_phi: 0.1506 | Regression_energy: 0.1369 | Regression_eta: 0.0799 | Regression_pt: 0.1406 | Regression_sin_phi: 0.1387 | LR: 9.43e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.3409s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank2 [2026-10-28 18:13:41,921] INFO : Step 8200/50000 rank2 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.2438 | Regression_cos_phi: 0.1152 | Regression_energy: 0.1183 | Regression_eta: 0.0890 | Regression_pt: 0.1251 | Regression_sin_phi: 0.1335 | LR: 9.42e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3250s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank2 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank2 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.3113 | Regression_cos_phi: 0.1357 | Regression_energy: 0.1270 | Regression_eta: 0.0910 | Regression_pt: 0.1318 | Regression_sin_phi: 0.1230 | LR: 9.40e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6372s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank2 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank2 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1131 | Classification_binary: 0.2981 | Regression_cos_phi: 0.1515 | Regression_energy: 0.1457 | Regression_eta: 0.0992 | Regression_pt: 0.1493 | Regression_sin_phi: 0.1647 | LR: 9.39e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1845s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank2 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank2 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1097 | Classification_binary: 0.2322 | Regression_cos_phi: 0.1182 | Regression_energy: 0.1315 | Regression_eta: 0.1122 | Regression_pt: 0.1340 | Regression_sin_phi: 0.1598 | LR: 9.37e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.5045s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank2 [2026-10-28 18:34:09,472] INFO : Step 8600/50000 rank2 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1212 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1580 | Regression_energy: 0.1494 | Regression_eta: 0.1010 | Regression_pt: 0.1522 | Regression_sin_phi: 0.1417 | LR: 9.36e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7741s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank2 [2026-10-28 18:39:14,579] INFO : Step 8700/50000 rank2 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2731 | Regression_cos_phi: 0.1266 | Regression_energy: 0.1480 | Regression_eta: 0.0951 | Regression_pt: 0.1492 | Regression_sin_phi: 0.1102 | LR: 9.34e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.7210s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank2 [2026-10-28 18:44:19,413] INFO : Step 8800/50000 rank2 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.2318 | Regression_cos_phi: 0.1185 | Regression_energy: 0.1454 | Regression_eta: 0.0907 | Regression_pt: 0.1491 | Regression_sin_phi: 0.1250 | LR: 9.33e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.4100s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank2 [2026-10-28 18:49:28,517] INFO : Step 8900/50000 rank2 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1082 | Classification_binary: 0.3352 | Regression_cos_phi: 0.1584 | Regression_energy: 0.1396 | Regression_eta: 0.1173 | Regression_pt: 0.1465 | Regression_sin_phi: 0.1575 | LR: 9.31e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7931s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank2 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank2 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.3560 | Regression_cos_phi: 0.1442 | Regression_energy: 0.1411 | Regression_eta: 0.1156 | Regression_pt: 0.1438 | Regression_sin_phi: 0.1852 | LR: 9.30e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank2 [2026-10-28 18:54:54,936] INFO : Running validation on rank2 +rank2 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:55:52,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:55:53,258] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 18:56:19,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:56:21,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:56:22,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:56:24,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:56:34,293] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:56:35,474] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 18:56:51,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:56:52,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:56:53,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:56:54,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:57:04,594] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:57:05,961] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 18:57:21,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:57:22,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:57:23,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:57:23,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:57:33,922] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 19:03:08,598] INFO : Step 9100/50000 rank2 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2819 | Regression_cos_phi: 0.0994 | Regression_energy: 0.1196 | Regression_eta: 0.0817 | Regression_pt: 0.1216 | Regression_sin_phi: 0.0941 | LR: 9.28e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.1691s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank2 [2026-10-28 19:08:18,223] INFO : Step 9200/50000 rank2 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0886 | Regression_energy: 0.1088 | Regression_eta: 0.0589 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0798 | LR: 9.27e-04 | DataLoad Time: 0.0251s | Model Forward Time: 6.5440s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank2 [2026-10-28 19:13:30,252] INFO : Step 9300/50000 rank2 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2689 | Regression_cos_phi: 0.1385 | Regression_energy: 0.1286 | Regression_eta: 0.0751 | Regression_pt: 0.1328 | Regression_sin_phi: 0.1176 | LR: 9.25e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3902s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank2 [2026-10-28 19:18:55,613] INFO : Step 9400/50000 rank2 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2669 | Regression_cos_phi: 0.1446 | Regression_energy: 0.1280 | Regression_eta: 0.0962 | Regression_pt: 0.1339 | Regression_sin_phi: 0.1173 | LR: 9.24e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2021s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank2 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank2 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2192 | Regression_cos_phi: 0.1269 | Regression_energy: 0.1301 | Regression_eta: 0.0911 | Regression_pt: 0.1348 | Regression_sin_phi: 0.1845 | LR: 9.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2845s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank2 [2026-10-28 19:29:34,438] INFO : Step 9600/50000 rank2 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2561 | Regression_cos_phi: 0.1265 | Regression_energy: 0.1245 | Regression_eta: 0.0884 | Regression_pt: 0.1301 | Regression_sin_phi: 0.1428 | LR: 9.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6043s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank2 [2026-10-28 19:34:53,790] INFO : Step 9700/50000 rank2 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.3318 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1240 | Regression_eta: 0.0899 | Regression_pt: 0.1292 | Regression_sin_phi: 0.1199 | LR: 9.19e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.9762s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank2 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank2 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0975 | Regression_energy: 0.1221 | Regression_eta: 0.0624 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0884 | LR: 9.17e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.5614s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank2 [2026-10-28 19:44:46,547] INFO : Step 9900/50000 rank2 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2214 | Regression_cos_phi: 0.1091 | Regression_energy: 0.1226 | Regression_eta: 0.0813 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0982 | LR: 9.16e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9295s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank2 [2026-10-28 19:49:37,631] INFO : Step 10000/50000 rank2 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2894 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1101 | Regression_eta: 0.0845 | Regression_pt: 0.1161 | Regression_sin_phi: 0.1332 | LR: 9.14e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.6247s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank2 [2026-10-28 19:49:38,690] INFO : Running validation on rank2 +rank2 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 19:50:35,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:50:35,626] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 19:51:01,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 19:51:02,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 19:51:03,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 19:51:04,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 19:51:14,980] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:51:15,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:51:16,369] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 19:51:32,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 19:51:33,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 19:51:34,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 19:51:35,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 19:51:45,085] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:51:46,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:51:46,687] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 19:52:02,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 19:52:03,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 19:52:04,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 19:52:05,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 19:52:15,169] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 19:57:27,968] INFO : Step 10100/50000 rank2 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2952 | Regression_cos_phi: 0.1205 | Regression_energy: 0.1228 | Regression_eta: 0.1069 | Regression_pt: 0.1277 | Regression_sin_phi: 0.1531 | LR: 9.12e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7469s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank2 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank2 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2084 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1176 | Regression_eta: 0.0789 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0964 | LR: 9.11e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6804s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank2 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank2 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0970 | Classification_binary: 0.3048 | Regression_cos_phi: 0.1413 | Regression_energy: 0.1257 | Regression_eta: 0.0967 | Regression_pt: 0.1288 | Regression_sin_phi: 0.1074 | LR: 9.09e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.1998s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank2 [2026-10-28 20:12:49,148] INFO : Step 10400/50000 rank2 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2858 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1273 | Regression_eta: 0.0841 | Regression_pt: 0.1307 | Regression_sin_phi: 0.1391 | LR: 9.07e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2217s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank2 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank2 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1841 | Regression_cos_phi: 0.1014 | Regression_energy: 0.1258 | Regression_eta: 0.0638 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0939 | LR: 9.06e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3232s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank2 [2026-10-28 20:23:17,055] INFO : Step 10600/50000 rank2 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2722 | Regression_cos_phi: 0.1375 | Regression_energy: 0.1085 | Regression_eta: 0.0851 | Regression_pt: 0.1134 | Regression_sin_phi: 0.1219 | LR: 9.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0933s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank2 [2026-10-28 20:28:22,710] INFO : Step 10700/50000 rank2 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2773 | Regression_cos_phi: 0.0994 | Regression_energy: 0.1152 | Regression_eta: 0.0816 | Regression_pt: 0.1213 | Regression_sin_phi: 0.1033 | LR: 9.02e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.8735s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank2 [2026-10-28 20:33:28,996] INFO : Step 10800/50000 rank2 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2282 | Regression_cos_phi: 0.0980 | Regression_energy: 0.1124 | Regression_eta: 0.0609 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0749 | LR: 9.00e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6834s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank2 [2026-10-28 20:38:44,061] INFO : Step 10900/50000 rank2 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2135 | Regression_cos_phi: 0.0888 | Regression_energy: 0.1101 | Regression_eta: 0.0488 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0894 | LR: 8.99e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7073s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank2 [2026-10-28 20:43:46,527] INFO : Step 11000/50000 rank2 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2773 | Regression_cos_phi: 0.1263 | Regression_energy: 0.1015 | Regression_eta: 0.0706 | Regression_pt: 0.1074 | Regression_sin_phi: 0.1277 | LR: 8.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4437s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank2 [2026-10-28 20:43:47,416] INFO : Running validation on rank2 +rank2 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 20:44:41,298] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 20:45:09,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 20:45:10,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 20:45:12,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 20:45:13,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 20:45:23,075] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 20:45:23,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:45:24,125] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 20:45:40,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 20:45:42,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 20:45:43,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 20:45:45,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 20:45:55,283] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 20:46:11,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 20:46:12,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 20:46:13,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 20:46:14,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 20:46:24,782] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 20:51:55,950] INFO : Step 11100/50000 rank2 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0974 | Regression_energy: 0.1039 | Regression_eta: 0.0676 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0955 | LR: 8.95e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8330s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank2 [2026-10-28 20:57:10,091] INFO : Step 11200/50000 rank2 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0996 | Regression_eta: 0.0506 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0938 | LR: 8.93e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8535s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank2 [2026-10-28 21:02:18,460] INFO : Step 11300/50000 rank2 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2715 | Regression_cos_phi: 0.1107 | Regression_energy: 0.1137 | Regression_eta: 0.0770 | Regression_pt: 0.1179 | Regression_sin_phi: 0.1098 | LR: 8.91e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2912s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank2 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank2 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2816 | Regression_cos_phi: 0.1592 | Regression_energy: 0.1118 | Regression_eta: 0.0770 | Regression_pt: 0.1179 | Regression_sin_phi: 0.1054 | LR: 8.89e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9849s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank2 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank2 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1960 | Regression_cos_phi: 0.1114 | Regression_energy: 0.0850 | Regression_eta: 0.0685 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0987 | LR: 8.88e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3387s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank2 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank2 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.2718 | Regression_cos_phi: 0.1450 | Regression_energy: 0.1079 | Regression_eta: 0.0770 | Regression_pt: 0.1125 | Regression_sin_phi: 0.1499 | LR: 8.86e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2581s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank2 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank2 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2989 | Regression_cos_phi: 0.0976 | Regression_energy: 0.1129 | Regression_eta: 0.0893 | Regression_pt: 0.1178 | Regression_sin_phi: 0.1463 | LR: 8.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank2 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank2 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0785 | Regression_energy: 0.1013 | Regression_eta: 0.0637 | Regression_pt: 0.1052 | Regression_sin_phi: 0.1114 | LR: 8.82e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5589s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank2 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank2 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0892 | Regression_eta: 0.0552 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0859 | LR: 8.80e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9853s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank2 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank2 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2746 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1014 | Regression_eta: 0.0759 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1016 | LR: 8.78e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0205s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank2 [2026-10-28 21:36:38,362] INFO : Running validation on rank2 +rank2 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:37:31,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 21:38:00,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:38:01,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:38:02,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:38:03,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:38:13,607] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:38:14,610] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 21:38:30,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:38:33,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:38:34,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:38:35,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:38:45,451] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 21:38:45,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:38:46,116] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 21:39:01,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:39:02,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:39:03,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:39:04,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:39:14,214] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 21:44:17,715] INFO : Step 12100/50000 rank2 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2600 | Regression_cos_phi: 0.1274 | Regression_energy: 0.1261 | Regression_eta: 0.0925 | Regression_pt: 0.1321 | Regression_sin_phi: 0.1657 | LR: 8.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1504s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank2 [2026-10-28 21:49:00,967] INFO : Step 12200/50000 rank2 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.1793 | Regression_cos_phi: 0.1060 | Regression_energy: 0.0965 | Regression_eta: 0.0827 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0770 | LR: 8.74e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5145s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank2 [2026-10-28 21:53:55,878] INFO : Step 12300/50000 rank2 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2364 | Regression_cos_phi: 0.0646 | Regression_energy: 0.1125 | Regression_eta: 0.0533 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0994 | LR: 8.72e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.6874s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank2 [2026-10-28 21:58:40,947] INFO : Step 12400/50000 rank2 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2753 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0916 | Regression_eta: 0.0635 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0854 | LR: 8.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2700s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank2 [2026-10-28 22:03:23,385] INFO : Step 12500/50000 rank2 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0663 | Regression_energy: 0.1144 | Regression_eta: 0.0704 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1039 | LR: 8.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6714s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank2 [2026-10-28 22:08:04,804] INFO : Step 12600/50000 rank2 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1824 | Regression_cos_phi: 0.1119 | Regression_energy: 0.0960 | Regression_eta: 0.1056 | Regression_pt: 0.1008 | Regression_sin_phi: 0.1128 | LR: 8.66e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.0966s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank2 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank2 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0942 | Regression_energy: 0.1005 | Regression_eta: 0.0791 | Regression_pt: 0.1027 | Regression_sin_phi: 0.1070 | LR: 8.64e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3622s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank2 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank2 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.2448 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0857 | Regression_eta: 0.0779 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0837 | LR: 8.62e-04 | DataLoad Time: 0.0248s | Model Forward Time: 6.0948s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank2 [2026-10-28 22:22:32,703] INFO : Step 12900/50000 rank2 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0913 | Regression_eta: 0.0576 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0777 | LR: 8.60e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6624s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank2 [2026-10-28 22:27:21,820] INFO : Step 13000/50000 rank2 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2693 | Regression_cos_phi: 0.1100 | Regression_energy: 0.0989 | Regression_eta: 0.0521 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0996 | LR: 8.58e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9255s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank2 [2026-10-28 22:27:23,211] INFO : Running validation on rank2 +rank2 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:28:28,169] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 22:28:54,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 22:28:55,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 22:28:56,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 22:28:57,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 22:29:07,246] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:29:08,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:29:08,604] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 22:29:24,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 22:29:26,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 22:29:27,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 22:29:27,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 22:29:38,003] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:29:38,493] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 22:29:54,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 22:29:54,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 22:29:55,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 22:29:56,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 22:30:06,774] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 22:35:15,708] INFO : Step 13100/50000 rank2 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2410 | Regression_cos_phi: 0.1182 | Regression_energy: 0.0932 | Regression_eta: 0.0616 | Regression_pt: 0.0966 | Regression_sin_phi: 0.1103 | LR: 8.56e-04 | DataLoad Time: 0.0219s | Model Forward Time: 6.4842s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank2 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank2 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2017 | Regression_cos_phi: 0.1122 | Regression_energy: 0.0880 | Regression_eta: 0.0679 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1089 | LR: 8.54e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6875s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank2 [2026-10-28 22:44:58,539] INFO : Step 13300/50000 rank2 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2473 | Regression_cos_phi: 0.1103 | Regression_energy: 0.0861 | Regression_eta: 0.0737 | Regression_pt: 0.0893 | Regression_sin_phi: 0.1189 | LR: 8.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6752s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank2 [2026-10-28 22:49:50,191] INFO : Step 13400/50000 rank2 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1053 | Classification_binary: 0.3117 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1337 | Regression_eta: 0.0957 | Regression_pt: 0.1384 | Regression_sin_phi: 0.1137 | LR: 8.50e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.6025s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank2 [2026-10-28 22:54:39,509] INFO : Step 13500/50000 rank2 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0801 | Regression_eta: 0.0851 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0988 | LR: 8.48e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4610s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank2 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank2 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2043 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0798 | Regression_eta: 0.0504 | Regression_pt: 0.0807 | Regression_sin_phi: 0.1189 | LR: 8.45e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8487s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank2 [2026-10-28 23:04:05,988] INFO : Step 13700/50000 rank2 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0929 | Classification_binary: 0.2914 | Regression_cos_phi: 0.1192 | Regression_energy: 0.0913 | Regression_eta: 0.0615 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0920 | LR: 8.43e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7310s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank2 [2026-10-28 23:09:16,293] INFO : Step 13800/50000 rank2 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2181 | Regression_cos_phi: 0.1278 | Regression_energy: 0.1086 | Regression_eta: 0.0713 | Regression_pt: 0.1136 | Regression_sin_phi: 0.1039 | LR: 8.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4790s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank2 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank2 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2343 | Regression_cos_phi: 0.0839 | Regression_energy: 0.1473 | Regression_eta: 0.0634 | Regression_pt: 0.1479 | Regression_sin_phi: 0.1037 | LR: 8.39e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2880s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank2 [2026-10-28 23:18:59,721] INFO : Step 14000/50000 rank2 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.2177 | Regression_cos_phi: 0.1073 | Regression_energy: 0.0838 | Regression_eta: 0.0517 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0894 | LR: 8.37e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1755s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank2 [2026-10-28 23:19:00,860] INFO : Running validation on rank2 +rank2 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:19:53,516] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-28 23:20:19,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 23:20:20,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 23:20:21,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 23:20:22,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 23:20:32,479] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 23:20:33,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:20:33,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-28 23:20:49,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 23:20:50,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 23:20:51,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 23:20:52,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 23:21:02,441] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 23:21:03,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:21:04,038] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-28 23:21:19,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 23:21:20,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 23:21:21,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 23:21:22,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 23:21:32,370] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 23:26:43,450] INFO : Step 14100/50000 rank2 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0802 | Regression_eta: 0.0757 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0640 | LR: 8.35e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.5310s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank2 [2026-10-28 23:31:31,721] INFO : Step 14200/50000 rank2 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1937 | Regression_cos_phi: 0.1105 | Regression_energy: 0.0839 | Regression_eta: 0.0663 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0673 | LR: 8.32e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6179s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank2 [2026-10-28 23:36:16,486] INFO : Step 14300/50000 rank2 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0897 | Regression_eta: 0.0647 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0924 | LR: 8.30e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6726s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank2 [2026-10-28 23:40:58,338] INFO : Step 14400/50000 rank2 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2671 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0857 | Regression_eta: 0.0583 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0789 | LR: 8.28e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7775s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank2 [2026-10-28 23:45:42,941] INFO : Step 14500/50000 rank2 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0850 | Regression_eta: 0.0672 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0873 | LR: 8.26e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.6355s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank2 [2026-10-28 23:50:27,494] INFO : Step 14600/50000 rank2 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0842 | Regression_eta: 0.0512 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0585 | LR: 8.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4498s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank2 [2026-10-28 23:55:12,844] INFO : Step 14700/50000 rank2 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2903 | Regression_cos_phi: 0.1227 | Regression_energy: 0.1008 | Regression_eta: 0.0761 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1114 | LR: 8.21e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6491s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank2 [2026-10-28 23:59:56,485] INFO : Step 14800/50000 rank2 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0870 | Regression_eta: 0.0559 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0907 | LR: 8.19e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.7610s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank2 [2026-10-29 00:04:46,437] INFO : Step 14900/50000 rank2 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0890 | Regression_eta: 0.0490 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0711 | LR: 8.17e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1952s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank2 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank2 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2396 | Regression_cos_phi: 0.1096 | Regression_energy: 0.0988 | Regression_eta: 0.0751 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0968 | LR: 8.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7487s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank2 [2026-10-29 00:09:44,171] INFO : Running validation on rank2 +rank2 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:10:38,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:10:39,315] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 00:11:05,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:11:07,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:11:08,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:11:09,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:11:19,322] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 00:11:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:11:20,773] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 00:11:36,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:11:37,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:11:38,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:11:39,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:11:49,768] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:11:50,984] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 00:12:06,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:12:07,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:12:08,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:12:09,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:12:19,134] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 00:17:30,255] INFO : Step 15100/50000 rank2 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0845 | Regression_eta: 0.0501 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0949 | LR: 8.12e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.3995s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank2 [2026-10-29 00:22:16,916] INFO : Step 15200/50000 rank2 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1715 | Regression_cos_phi: 0.1184 | Regression_energy: 0.0886 | Regression_eta: 0.0601 | Regression_pt: 0.0909 | Regression_sin_phi: 0.1420 | LR: 8.10e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2232s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank2 [2026-10-29 00:27:05,720] INFO : Step 15300/50000 rank2 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2311 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0909 | Regression_eta: 0.0579 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0743 | LR: 8.08e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3175s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank2 [2026-10-29 00:31:57,727] INFO : Step 15400/50000 rank2 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2787 | Regression_cos_phi: 0.1053 | Regression_energy: 0.0959 | Regression_eta: 0.0726 | Regression_pt: 0.0989 | Regression_sin_phi: 0.1323 | LR: 8.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6646s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank2 [2026-10-29 00:36:52,791] INFO : Step 15500/50000 rank2 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0782 | Regression_eta: 0.0408 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0687 | LR: 8.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4234s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank2 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank2 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0642 | Regression_eta: 0.0355 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0603 | LR: 8.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9294s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank2 [2026-10-29 00:46:43,872] INFO : Step 15700/50000 rank2 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2387 | Regression_cos_phi: 0.1350 | Regression_energy: 0.0839 | Regression_eta: 0.0644 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0710 | LR: 7.98e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7564s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank2 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank2 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0768 | Regression_eta: 0.0779 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0895 | LR: 7.96e-04 | DataLoad Time: 0.0241s | Model Forward Time: 5.0242s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank2 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank2 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1773 | Regression_cos_phi: 0.1116 | Regression_energy: 0.0773 | Regression_eta: 0.0537 | Regression_pt: 0.0778 | Regression_sin_phi: 0.1570 | LR: 7.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank2 [2026-10-29 01:01:16,824] INFO : Step 16000/50000 rank2 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2177 | Regression_cos_phi: 0.1190 | Regression_energy: 0.0812 | Regression_eta: 0.0611 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0829 | LR: 7.91e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.3020s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank2 [2026-10-29 01:01:17,883] INFO : Running validation on rank2 +rank2 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 01:02:10,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:02:11,391] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 01:02:39,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 01:02:40,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 01:02:42,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 01:02:44,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 01:02:54,509] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-29 01:02:55,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:02:55,637] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 01:03:11,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 01:03:12,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 01:03:13,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 01:03:15,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 01:03:25,079] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 01:03:25,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:03:26,369] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 01:03:41,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 01:03:42,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 01:03:43,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 01:03:44,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 01:03:54,473] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank2 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1995 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0744 | Regression_eta: 0.0546 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0672 | LR: 7.89e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0651s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank2 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank2 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0737 | Regression_eta: 0.0804 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0653 | LR: 7.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2113s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank2 [2026-10-29 01:19:03,894] INFO : Step 16300/50000 rank2 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2294 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0797 | Regression_eta: 0.0662 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0931 | LR: 7.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.2091s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank2 [2026-10-29 01:23:56,184] INFO : Step 16400/50000 rank2 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2237 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0725 | Regression_eta: 0.0623 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0713 | LR: 7.81e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6848s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank2 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank2 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0709 | Regression_eta: 0.0526 | Regression_pt: 0.0726 | Regression_sin_phi: 0.1012 | LR: 7.79e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3095s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank2 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank2 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0672 | Regression_eta: 0.0585 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0489 | LR: 7.77e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1717s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank2 [2026-10-29 01:38:38,619] INFO : Step 16700/50000 rank2 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2404 | Regression_cos_phi: 0.1275 | Regression_energy: 0.0888 | Regression_eta: 0.1365 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0896 | LR: 7.74e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.9114s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank2 [2026-10-29 01:43:30,910] INFO : Step 16800/50000 rank2 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2452 | Regression_cos_phi: 0.1174 | Regression_energy: 0.1061 | Regression_eta: 0.0654 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0871 | LR: 7.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.6004s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank2 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank2 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0860 | Regression_eta: 0.0493 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0845 | LR: 7.69e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3817s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank2 [2026-10-29 01:53:19,524] INFO : Step 17000/50000 rank2 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0749 | Regression_eta: 0.0548 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0875 | LR: 7.67e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.6704s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank2 [2026-10-29 01:53:20,626] INFO : Running validation on rank2 +rank2 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 01:54:26,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:54:27,390] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 01:55:04,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 01:55:05,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 01:55:06,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 01:55:07,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 01:55:17,859] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:55:18,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 01:55:34,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 01:55:35,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 01:55:36,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 01:55:37,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 01:55:47,974] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:55:50,413] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 01:56:05,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 01:56:06,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 01:56:07,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 01:56:08,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 01:56:18,382] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 02:01:32,243] INFO : Step 17100/50000 rank2 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0706 | Regression_eta: 0.0579 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0719 | LR: 7.64e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.1165s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank2 [2026-10-29 02:06:24,490] INFO : Step 17200/50000 rank2 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0766 | Regression_eta: 0.0584 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0673 | LR: 7.62e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.6094s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank2 [2026-10-29 02:11:06,510] INFO : Step 17300/50000 rank2 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0900 | Regression_energy: 0.0842 | Regression_eta: 0.0447 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0707 | LR: 7.59e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7526s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank2 [2026-10-29 02:16:07,514] INFO : Step 17400/50000 rank2 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.2236 | Regression_cos_phi: 0.1148 | Regression_energy: 0.0694 | Regression_eta: 0.0571 | Regression_pt: 0.0730 | Regression_sin_phi: 0.1085 | LR: 7.57e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5469s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank2 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank2 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0704 | Regression_eta: 0.0433 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0441 | LR: 7.54e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.2406s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank2 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank2 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0874 | Regression_eta: 0.0649 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0678 | LR: 7.52e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.4721s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank2 [2026-10-29 02:30:47,051] INFO : Step 17700/50000 rank2 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0885 | Regression_eta: 0.0471 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0680 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7694s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank2 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank2 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0759 | Regression_eta: 0.0507 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0604 | LR: 7.47e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0216s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank2 [2026-10-29 02:40:30,747] INFO : Step 17900/50000 rank2 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0772 | Regression_eta: 0.0425 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0700 | LR: 7.44e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5937s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank2 [2026-10-29 02:45:16,090] INFO : Step 18000/50000 rank2 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0639 | Regression_eta: 0.0447 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0646 | LR: 7.42e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6581s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank2 [2026-10-29 02:45:17,190] INFO : Running validation on rank2 +rank2 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:45:17,191] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:46:12,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:46:13,322] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 02:46:39,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:46:40,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:46:41,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:46:42,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:46:52,605] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:46:53,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 02:47:09,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:47:10,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:47:12,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:47:14,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:47:24,513] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-29 02:47:25,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:47:25,822] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 02:47:41,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:47:42,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:47:43,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:47:44,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:47:54,498] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank2 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2507 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0718 | Regression_eta: 0.0565 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0735 | LR: 7.39e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7380s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank2 [2026-10-29 02:58:08,645] INFO : Step 18200/50000 rank2 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0822 | Regression_eta: 0.0470 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0634 | LR: 7.36e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.9648s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank2 [2026-10-29 03:02:55,065] INFO : Step 18300/50000 rank2 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1568 | Regression_cos_phi: 0.1073 | Regression_energy: 0.0716 | Regression_eta: 0.0885 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0864 | LR: 7.34e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.5924s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank2 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank2 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.2357 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0739 | Regression_eta: 0.0536 | Regression_pt: 0.0772 | Regression_sin_phi: 0.1120 | LR: 7.31e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7136s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank2 [2026-10-29 03:12:43,564] INFO : Step 18500/50000 rank2 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0992 | Regression_energy: 0.0783 | Regression_eta: 0.0674 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0810 | LR: 7.29e-04 | DataLoad Time: 0.0099s | Model Forward Time: 5.3872s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank2 [2026-10-29 03:17:39,475] INFO : Step 18600/50000 rank2 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0808 | Regression_eta: 0.0393 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0460 | LR: 7.26e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.4733s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank2 [2026-10-29 03:22:31,158] INFO : Step 18700/50000 rank2 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0954 | Regression_eta: 0.0554 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0751 | LR: 7.24e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4799s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank2 [2026-10-29 03:27:37,244] INFO : Step 18800/50000 rank2 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0660 | Regression_eta: 0.0567 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0620 | LR: 7.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.9527s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank2 [2026-10-29 03:32:26,943] INFO : Step 18900/50000 rank2 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0643 | Regression_eta: 0.0435 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0742 | LR: 7.18e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5367s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank2 [2026-10-29 03:37:06,569] INFO : Step 19000/50000 rank2 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0848 | Regression_eta: 0.0831 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0992 | LR: 7.16e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5317s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank2 [2026-10-29 03:37:07,713] INFO : Running validation on rank2 +rank2 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 03:38:00,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:38:00,821] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 03:38:27,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 03:38:28,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 03:38:29,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 03:38:30,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 03:38:40,431] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 03:38:40,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:38:41,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 03:38:57,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 03:38:58,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 03:38:59,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 03:39:00,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 03:39:10,594] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:39:11,227] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 03:39:26,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 03:39:27,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 03:39:28,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 03:39:28,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 03:39:38,924] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank2 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.2471 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0658 | Regression_eta: 0.0525 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0662 | LR: 7.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9863s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank2 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank2 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0654 | Regression_eta: 0.0563 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0652 | LR: 7.10e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5599s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank2 [2026-10-29 03:54:09,159] INFO : Step 19300/50000 rank2 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0755 | Regression_eta: 0.0498 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0616 | LR: 7.08e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3958s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank2 [2026-10-29 03:58:51,217] INFO : Step 19400/50000 rank2 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2396 | Regression_cos_phi: 0.0900 | Regression_energy: 0.0774 | Regression_eta: 0.0618 | Regression_pt: 0.0796 | Regression_sin_phi: 0.1110 | LR: 7.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6157s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank2 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank2 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2170 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0655 | Regression_eta: 0.0557 | Regression_pt: 0.0690 | Regression_sin_phi: 0.1024 | LR: 7.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7166s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank2 [2026-10-29 04:08:12,466] INFO : Step 19600/50000 rank2 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0623 | Regression_eta: 0.0520 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0966 | LR: 7.00e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.4189s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank2 [2026-10-29 04:12:52,347] INFO : Step 19700/50000 rank2 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0796 | Regression_eta: 0.0409 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0660 | LR: 6.97e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3572s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank2 [2026-10-29 04:17:28,314] INFO : Step 19800/50000 rank2 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1895 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0636 | Regression_eta: 0.0426 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0655 | LR: 6.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0448s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank2 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank2 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1754 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0760 | Regression_eta: 0.0635 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0927 | LR: 6.92e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.3168s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank2 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank2 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0713 | Regression_energy: 0.1010 | Regression_eta: 0.0466 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0658 | LR: 6.89e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6556s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank2 [2026-10-29 04:26:52,970] INFO : Running validation on rank2 +rank2 [2026-10-29 04:26:52,971] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:26:52,971] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:27:46,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:27:46,653] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 04:28:14,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:28:15,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:28:16,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:28:17,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:28:27,346] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:28:28,501] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 04:28:44,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:28:45,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:28:47,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:28:48,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:28:58,098] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:28:59,249] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 04:29:14,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:29:15,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:29:16,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:29:17,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:29:27,445] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 04:34:27,055] INFO : Step 20100/50000 rank2 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0651 | Regression_eta: 0.0381 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0574 | LR: 6.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.3254s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank2 [2026-10-29 04:39:08,147] INFO : Step 20200/50000 rank2 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0766 | Regression_eta: 0.0385 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0646 | LR: 6.84e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6157s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank2 [2026-10-29 04:43:47,103] INFO : Step 20300/50000 rank2 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0762 | Regression_eta: 0.0473 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0972 | LR: 6.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4960s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank2 [2026-10-29 04:48:26,731] INFO : Step 20400/50000 rank2 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0726 | Regression_eta: 0.0372 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0622 | LR: 6.78e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4430s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank2 [2026-10-29 04:53:08,466] INFO : Step 20500/50000 rank2 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0657 | Regression_eta: 0.0581 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0674 | LR: 6.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0622s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank2 [2026-10-29 04:57:53,294] INFO : Step 20600/50000 rank2 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0234 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0738 | Regression_eta: 0.0376 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0798 | LR: 6.73e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7517s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank2 [2026-10-29 05:02:47,890] INFO : Step 20700/50000 rank2 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0677 | Regression_eta: 0.0520 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0819 | LR: 6.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6386s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank2 [2026-10-29 05:07:34,619] INFO : Step 20800/50000 rank2 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2120 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0606 | Regression_eta: 0.0459 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0600 | LR: 6.67e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.8137s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank2 [2026-10-29 05:12:18,486] INFO : Step 20900/50000 rank2 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0635 | Regression_eta: 0.0482 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0730 | LR: 6.65e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6288s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank2 [2026-10-29 05:17:03,998] INFO : Step 21000/50000 rank2 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0691 | Regression_eta: 0.0451 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0786 | LR: 6.62e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank2 [2026-10-29 05:17:04,884] INFO : Running validation on rank2 +rank2 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 05:17:58,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:17:58,802] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 05:18:23,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 05:18:24,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 05:18:25,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 05:18:26,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 05:18:36,370] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 05:18:36,641] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:18:37,119] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 05:18:52,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 05:18:53,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 05:18:54,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 05:18:55,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 05:19:05,918] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 05:19:06,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:19:07,300] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 05:19:22,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 05:19:23,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 05:19:24,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 05:19:25,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 05:19:35,319] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 05:24:38,582] INFO : Step 21100/50000 rank2 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2225 | Regression_cos_phi: 0.1083 | Regression_energy: 0.0686 | Regression_eta: 0.0666 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0755 | LR: 6.59e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3791s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank2 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank2 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 05:29:24,523] INFO : Step 21200/50000 rank2 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0589 | Regression_eta: 0.0403 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0631 | LR: 6.56e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6428s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank2 [2026-10-29 05:34:39,116] INFO : Step 21300/50000 rank2 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1976 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0736 | Regression_eta: 0.0515 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0625 | LR: 6.54e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6737s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank2 [2026-10-29 05:39:29,689] INFO : Step 21400/50000 rank2 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0713 | Regression_eta: 0.0493 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0729 | LR: 6.51e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3947s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank2 [2026-10-29 05:44:17,736] INFO : Step 21500/50000 rank2 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0734 | Regression_eta: 0.0596 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0793 | LR: 6.48e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3093s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank2 [2026-10-29 05:49:11,032] INFO : Step 21600/50000 rank2 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0266 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0509 | Regression_eta: 0.0427 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0580 | LR: 6.45e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5077s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank2 [2026-10-29 05:53:59,566] INFO : Step 21700/50000 rank2 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0545 | Regression_eta: 0.0439 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0769 | LR: 6.43e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.5565s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank2 [2026-10-29 05:58:52,461] INFO : Step 21800/50000 rank2 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0579 | Regression_eta: 0.0377 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0450 | LR: 6.40e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.1507s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank2 [2026-10-29 06:03:49,408] INFO : Step 21900/50000 rank2 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0629 | Regression_energy: 0.1062 | Regression_eta: 0.0360 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0611 | LR: 6.37e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8518s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank2 [2026-10-29 06:08:50,713] INFO : Step 22000/50000 rank2 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0767 | Regression_eta: 0.0429 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0497 | LR: 6.34e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.4716s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank2 [2026-10-29 06:08:51,802] INFO : Running validation on rank2 +rank2 [2026-10-29 06:08:51,803] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 06:08:51,803] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 06:09:45,573] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:09:46,124] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 06:10:12,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 06:10:13,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 06:10:14,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 06:10:16,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 06:10:26,226] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 06:10:26,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 06:10:43,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 06:10:44,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 06:10:45,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 06:10:46,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 06:10:56,398] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:10:56,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:10:57,444] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 06:11:13,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 06:11:13,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 06:11:14,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 06:11:15,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 06:11:25,566] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank2 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.2570 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0731 | Regression_eta: 0.0525 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0808 | LR: 6.32e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.1904s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank2 [2026-10-29 06:21:31,731] INFO : Step 22200/50000 rank2 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0894 | Regression_eta: 0.0377 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0700 | LR: 6.29e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.2928s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank2 [2026-10-29 06:26:33,414] INFO : Step 22300/50000 rank2 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1658 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0626 | Regression_eta: 0.0483 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0666 | LR: 6.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7531s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank2 [2026-10-29 06:31:29,808] INFO : Step 22400/50000 rank2 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0628 | Regression_eta: 0.0542 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0739 | LR: 6.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.3997s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank2 [2026-10-29 06:36:31,515] INFO : Step 22500/50000 rank2 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0708 | Regression_eta: 0.0441 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0519 | LR: 6.20e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2551s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank2 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank2 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0670 | Regression_eta: 0.0574 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0624 | LR: 6.18e-04 | DataLoad Time: 0.0252s | Model Forward Time: 4.3124s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank2 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank2 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0512 | Regression_eta: 0.0414 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0967 | LR: 6.15e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9431s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank2 [2026-10-29 06:51:26,820] INFO : Step 22800/50000 rank2 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0851 | Regression_eta: 0.0531 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0758 | LR: 6.12e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.4039s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank2 [2026-10-29 06:56:23,354] INFO : Step 22900/50000 rank2 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0946 | Regression_eta: 0.0357 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0707 | LR: 6.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8111s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank2 [2026-10-29 07:01:33,063] INFO : Step 23000/50000 rank2 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0682 | Regression_eta: 0.0564 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0548 | LR: 6.06e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.4757s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank2 [2026-10-29 07:01:34,091] INFO : Running validation on rank2 +rank2 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:02:28,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 07:02:49,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:02:50,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:02:51,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:02:52,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:03:02,603] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:03:03,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:03:03,686] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 07:03:19,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:03:20,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:03:21,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:03:22,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:03:32,987] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:03:33,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:03:34,170] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 07:03:49,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:03:50,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:03:51,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:03:52,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:04:02,230] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 07:09:13,970] INFO : Step 23100/50000 rank2 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0673 | Regression_eta: 0.0595 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0826 | LR: 6.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1794s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank2 [2026-10-29 07:14:12,718] INFO : Step 23200/50000 rank2 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0624 | Regression_eta: 0.0412 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0724 | LR: 6.01e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6131s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank2 [2026-10-29 07:19:05,797] INFO : Step 23300/50000 rank2 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0936 | Regression_eta: 0.0353 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0588 | LR: 5.98e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8576s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank2 [2026-10-29 07:24:01,131] INFO : Step 23400/50000 rank2 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0572 | Regression_eta: 0.0503 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0693 | LR: 5.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0089s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank2 [2026-10-29 07:29:14,710] INFO : Step 23500/50000 rank2 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0689 | Regression_eta: 0.0697 | Regression_pt: 0.0700 | Regression_sin_phi: 0.1046 | LR: 5.92e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8325s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank2 [2026-10-29 07:34:22,319] INFO : Step 23600/50000 rank2 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0809 | Regression_eta: 0.0381 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0747 | LR: 5.90e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.3904s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank2 [2026-10-29 07:39:40,891] INFO : Step 23700/50000 rank2 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0611 | Regression_eta: 0.0414 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0533 | LR: 5.87e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.8157s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank2 [2026-10-29 07:44:46,683] INFO : Step 23800/50000 rank2 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0617 | Regression_eta: 0.0397 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0713 | LR: 5.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8768s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank2 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank2 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0721 | Regression_eta: 0.0432 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0540 | LR: 5.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.8179s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank2 [2026-10-29 07:54:58,898] INFO : Step 24000/50000 rank2 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0618 | Regression_eta: 0.0453 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0789 | LR: 5.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8546s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank2 [2026-10-29 07:54:59,832] INFO : Running validation on rank2 +rank2 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:54:59,833] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:55:54,112] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 07:56:21,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:56:22,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:56:23,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:56:24,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:56:34,202] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:56:35,055] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 07:56:51,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:56:52,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:56:53,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:56:54,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:57:04,210] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:57:05,694] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 07:57:21,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:57:22,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:57:23,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:57:24,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:57:34,107] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 08:03:04,653] INFO : Step 24100/50000 rank2 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0710 | Regression_eta: 0.0470 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0846 | LR: 5.75e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.5914s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank2 [2026-10-29 08:07:59,794] INFO : Step 24200/50000 rank2 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0702 | Regression_eta: 0.0381 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0603 | LR: 5.73e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0941s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank2 [2026-10-29 08:13:18,506] INFO : Step 24300/50000 rank2 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0516 | Regression_eta: 0.0315 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0428 | LR: 5.70e-04 | DataLoad Time: 0.0208s | Model Forward Time: 7.4897s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank2 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank2 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0669 | Regression_eta: 0.0468 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0493 | LR: 5.67e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1579s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank2 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank2 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0615 | Regression_eta: 0.0586 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0791 | LR: 5.64e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.5210s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank2 [2026-10-29 08:28:18,514] INFO : Step 24600/50000 rank2 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0585 | Regression_eta: 0.0479 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0705 | LR: 5.61e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7045s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank2 [2026-10-29 08:33:21,874] INFO : Step 24700/50000 rank2 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0698 | Regression_eta: 0.0510 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0988 | LR: 5.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9284s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank2 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank2 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2192 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0577 | Regression_eta: 0.0544 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0851 | LR: 5.56e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.2077s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank2 [2026-10-29 08:43:21,523] INFO : Step 24900/50000 rank2 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0655 | Regression_eta: 0.0367 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0540 | LR: 5.53e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4717s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank2 [2026-10-29 08:48:25,835] INFO : Step 25000/50000 rank2 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0671 | Regression_eta: 0.0431 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0688 | LR: 5.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9523s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank2 [2026-10-29 08:48:26,771] INFO : Running validation on rank2 +rank2 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 08:49:20,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:49:21,192] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 08:49:51,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 08:49:52,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 08:49:54,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 08:49:55,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 08:50:05,458] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 08:50:06,216] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:50:06,730] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 08:50:23,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 08:50:24,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 08:50:25,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 08:50:26,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 08:50:36,070] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:50:36,988] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 08:50:52,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 08:50:53,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 08:50:54,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 08:50:55,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 08:51:05,202] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 08:56:18,537] INFO : Step 25100/50000 rank2 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2109 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0730 | Regression_eta: 0.0585 | Regression_pt: 0.0754 | Regression_sin_phi: 0.1194 | LR: 5.47e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.3781s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank2 [2026-10-29 09:01:01,590] INFO : Step 25200/50000 rank2 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0593 | Regression_eta: 0.0370 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0445 | LR: 5.44e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8755s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank2 [2026-10-29 09:05:50,664] INFO : Step 25300/50000 rank2 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1083 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0536 | Regression_eta: 0.0233 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0411 | LR: 5.42e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9793s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank2 [2026-10-29 09:10:30,564] INFO : Step 25400/50000 rank2 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0653 | Regression_eta: 0.0501 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0658 | LR: 5.39e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.1682s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank2 [2026-10-29 09:15:11,238] INFO : Step 25500/50000 rank2 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0539 | Regression_eta: 0.0387 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0605 | LR: 5.36e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank2 [2026-10-29 09:19:56,596] INFO : Step 25600/50000 rank2 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0582 | Regression_eta: 0.0324 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0503 | LR: 5.33e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.9355s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank2 [2026-10-29 09:24:33,864] INFO : Step 25700/50000 rank2 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0638 | Regression_eta: 0.0328 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0541 | LR: 5.30e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2131s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank2 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank2 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2298 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0802 | Regression_eta: 0.0680 | Regression_pt: 0.0843 | Regression_sin_phi: 0.1060 | LR: 5.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1670s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank2 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank2 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0566 | Regression_eta: 0.0527 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0700 | LR: 5.25e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.7725s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank2 [2026-10-29 09:39:03,491] INFO : Step 26000/50000 rank2 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0592 | Regression_eta: 0.0527 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0610 | LR: 5.22e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7474s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank2 [2026-10-29 09:39:04,383] INFO : Running validation on rank2 +rank2 [2026-10-29 09:39:04,383] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:39:04,383] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:39:55,910] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 09:40:23,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:40:24,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:40:25,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:40:26,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:40:36,465] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:40:37,505] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 09:40:54,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:40:56,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:40:57,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:40:58,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:41:08,470] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-29 09:41:08,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:44:09,566] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 09:44:24,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:44:26,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:44:27,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:44:27,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:44:37,837] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 09:50:29,554] INFO : Step 26100/50000 rank2 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0667 | Regression_eta: 0.0489 | Regression_pt: 0.0674 | Regression_sin_phi: 0.1129 | LR: 5.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1779s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank2 [2026-10-29 09:55:15,272] INFO : Step 26200/50000 rank2 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0578 | Regression_eta: 0.0355 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0957 | LR: 5.16e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8128s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank2 [2026-10-29 09:59:57,744] INFO : Step 26300/50000 rank2 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0649 | Regression_eta: 0.0358 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0612 | LR: 5.13e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.9312s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank2 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank2 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0514 | Regression_eta: 0.0438 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0529 | LR: 5.10e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.1492s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank2 [2026-10-29 10:09:23,739] INFO : Step 26500/50000 rank2 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0662 | Regression_eta: 0.0485 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0534 | LR: 5.08e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.0948s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank2 [2026-10-29 10:14:05,781] INFO : Step 26600/50000 rank2 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0496 | Regression_eta: 0.0318 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0586 | LR: 5.05e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6619s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank2 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank2 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0673 | Regression_eta: 0.0379 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0656 | LR: 5.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2153s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank2 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank2 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0588 | Regression_eta: 0.0410 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0721 | LR: 4.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5299s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank2 [2026-10-29 10:28:51,387] INFO : Step 26900/50000 rank2 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0539 | Regression_eta: 0.0339 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0549 | LR: 4.96e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5658s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank2 [2026-10-29 10:33:37,219] INFO : Step 27000/50000 rank2 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0628 | Regression_eta: 0.0380 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0614 | LR: 4.94e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.7011s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank2 [2026-10-29 10:33:38,215] INFO : Running validation on rank2 +rank2 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 10:34:33,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:34:34,220] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 10:35:00,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 10:35:01,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 10:35:02,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 10:35:03,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 10:35:13,200] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 10:35:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:35:14,237] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 10:35:30,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 10:35:31,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 10:35:32,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 10:35:33,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 10:35:43,855] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 10:35:44,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:35:45,237] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 10:36:00,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 10:36:01,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 10:36:02,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 10:36:03,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 10:36:13,501] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 10:41:28,285] INFO : Step 27100/50000 rank2 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0629 | Regression_eta: 0.0396 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0793 | LR: 4.91e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.6719s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank2 [2026-10-29 10:46:22,293] INFO : Step 27200/50000 rank2 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0638 | Regression_eta: 0.0551 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0736 | LR: 4.88e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8269s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank2 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank2 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0576 | Regression_eta: 0.0404 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0536 | LR: 4.85e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.6697s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank2 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank2 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0509 | Regression_eta: 0.0258 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0560 | LR: 4.82e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9752s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank2 [2026-10-29 11:01:35,254] INFO : Step 27500/50000 rank2 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1958 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0522 | Regression_eta: 0.0555 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0579 | LR: 4.80e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.3142s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank2 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank2 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0462 | Regression_eta: 0.0327 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0460 | LR: 4.77e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.3337s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank2 [2026-10-29 11:13:33,056] INFO : Step 27700/50000 rank2 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0653 | Regression_eta: 0.0395 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0414 | LR: 4.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7941s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank2 [2026-10-29 11:18:30,826] INFO : Step 27800/50000 rank2 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0466 | Regression_eta: 0.0370 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0435 | LR: 4.71e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0113s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank2 [2026-10-29 11:23:38,362] INFO : Step 27900/50000 rank2 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0522 | Regression_eta: 0.0391 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0525 | LR: 4.68e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8872s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank2 [2026-10-29 11:28:54,878] INFO : Step 28000/50000 rank2 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0553 | Regression_eta: 0.0463 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0824 | LR: 4.66e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.9083s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank2 [2026-10-29 11:28:55,940] INFO : Running validation on rank2 +rank2 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 11:29:47,978] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:29:48,521] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 11:30:14,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 11:30:15,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 11:30:16,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 11:30:17,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 11:30:27,701] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:30:28,727] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 11:30:44,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 11:30:45,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 11:30:46,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 11:30:47,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 11:30:57,676] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 11:30:57,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 11:31:13,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 11:31:14,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 11:31:15,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 11:31:16,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 11:31:26,323] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 11:37:01,722] INFO : Step 28100/50000 rank2 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0503 | Regression_eta: 0.0358 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0630 | LR: 4.63e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.4999s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank2 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank2 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0524 | Regression_eta: 0.0337 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0746 | LR: 4.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.0740s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank2 [2026-10-29 11:47:09,276] INFO : Step 28300/50000 rank2 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0715 | Regression_eta: 0.0430 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0779 | LR: 4.57e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.0928s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank2 [2026-10-29 11:52:05,667] INFO : Step 28400/50000 rank2 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1161 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1398 | Regression_eta: 0.0743 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0845 | LR: 4.55e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5902s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank2 [2026-10-29 11:56:56,840] INFO : Step 28500/50000 rank2 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0569 | Regression_eta: 0.0440 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0697 | LR: 4.52e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9878s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank2 [2026-10-29 12:01:46,928] INFO : Step 28600/50000 rank2 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0432 | Regression_eta: 0.0435 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0492 | LR: 4.49e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6704s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank2 [2026-10-29 12:06:40,004] INFO : Step 28700/50000 rank2 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0611 | Regression_eta: 0.0460 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0712 | LR: 4.46e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.5118s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank2 [2026-10-29 12:11:31,023] INFO : Step 28800/50000 rank2 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0645 | Regression_eta: 0.0334 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0734 | LR: 4.44e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.9312s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank2 [2026-10-29 12:16:17,097] INFO : Step 28900/50000 rank2 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0543 | Regression_eta: 0.0468 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0740 | LR: 4.41e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7006s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank2 [2026-10-29 12:21:10,133] INFO : Step 29000/50000 rank2 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1103 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0609 | Regression_eta: 0.0337 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0402 | LR: 4.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9914s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank2 [2026-10-29 12:21:11,218] INFO : Running validation on rank2 +rank2 [2026-10-29 12:21:11,219] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 12:21:11,219] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 12:22:31,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 12:22:32,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 12:22:33,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 12:22:34,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 12:22:44,989] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 12:22:45,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:22:45,975] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 12:23:01,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 12:23:02,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 12:23:03,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 12:23:04,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 12:23:14,836] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 12:23:15,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:23:16,444] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 12:23:31,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 12:23:32,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 12:23:33,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 12:23:34,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 12:23:44,437] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 12:28:57,787] INFO : Step 29100/50000 rank2 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0491 | Regression_eta: 0.0414 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0794 | LR: 4.35e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1721s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank2 [2026-10-29 12:33:43,194] INFO : Step 29200/50000 rank2 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0518 | Regression_eta: 0.0860 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0959 | LR: 4.33e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5666s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank2 [2026-10-29 14:49:26,348] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-29 14:49:26,357] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank2 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-29 14:49:54,088] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank2 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank2 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank2 [2026-10-29 14:49:54,132] INFO : Initializing backbone layer 2 +rank2 [2026-10-29 14:49:54,141] INFO : Initializing backbone layer 3 +rank2 [2026-10-29 14:49:54,149] INFO : Initializing backbone layer 4 +rank2 [2026-10-29 14:49:54,157] INFO : Initializing backbone layer 5 +rank2 [2026-10-29 14:49:54,166] INFO : Backbone initialization took 0.08s +rank2 [2026-10-29 14:49:54,166] INFO : backbone parameters: 7901376 +rank2 [2026-10-29 14:49:54,166] INFO : Initializing output DNNs +rank2 [2026-10-29 14:49:54,202] INFO : Output DNNs initialization took 0.04s +rank2 [2026-10-29 14:49:54,202] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-29 14:49:54,202] INFO : jagged_attention enabled=False +rank2 [2026-10-29 14:49:54,202] INFO : flash_attn_varlen enabled=False +rank2 [2026-10-29 14:49:54,203] INFO : task_queries enabled=False +rank2 [2026-10-29 14:49:54,203] INFO : classification_norm parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : regression_norm parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : classification_query parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : regression_query parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : classification_readout parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : regression_readout parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : output_mode=set +rank2 [2026-10-29 14:49:54,203] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-29 14:49:54,203] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_pid parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_pt parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_eta parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-29 14:49:54,203] INFO : nn_energy parameters: 0 +rank2 [2026-10-29 14:49:54,204] INFO : Total MLPF parameters: 12786964 +rank2 [2026-10-29 14:49:54,204] INFO : MLPF __init__ done +rank2 [2026-10-29 14:49:54,204] INFO : Instantiated model +rank2 [2026-10-29 14:49:54,204] INFO : Moving model to device rank=2 +rank2 [2026-10-29 14:49:54,227] INFO : Moved model to device rank=2 +rank2 [2026-10-29 14:49:54,229] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-29 14:49:54,791] INFO : Restoring training from step 29001 +rank2 [2026-10-29 14:49:54,799] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank2 [2026-10-29 14:49:54,800] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-29 14:49:55,511] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank2 [2026-10-29 14:49:55,511] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-29 14:49:55,512] INFO : Trainable parameters: 12786964 +rank2 [2026-10-29 14:49:55,513] INFO : Non-trainable parameters: 23268 +rank2 [2026-10-29 14:49:55,513] INFO : Total parameters: 12810232 +rank2 [2026-10-29 14:49:55,516] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-29 14:49:55,516] INFO : Getting dataloaders +rank2 [2026-10-29 14:49:55,516] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:01,389] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:02,344] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:03,543] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-29 14:50:03,544] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-29 14:50:03,544] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:03,812] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:04,069] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:50:04,327] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-29 14:50:04,327] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-29 14:50:18,091] INFO : Got dataloaders +rank2 [2026-10-29 14:50:18,092] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank2 [2026-10-29 14:50:18,092] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank2 [2026-10-29 14:50:18,092] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank2 [2026-10-29 14:50:18,092] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank2 [2026-10-29 14:50:19,632] INFO : loader split=train rank=2 len=21108 +rank2 [2026-10-29 14:50:19,632] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-29 14:50:19,632] INFO : Starting training from step 29001 to 50000 on rank 2 of 8 +rank2 [2026-10-29 14:50:19,632] INFO : Creating train_iterator +rank2 [2026-10-29 14:50:19,632] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank2 [2026-10-29 14:50:33,984] INFO : Created train_iterator +rank2 [2026-10-29 14:55:12,919] INFO : Step 29100/50000 rank2 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0546 | Regression_eta: 0.0424 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0556 | LR: 4.35e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7151s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank2 [2026-10-29 14:59:46,788] INFO : Step 29200/50000 rank2 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0459 | Regression_eta: 0.1201 | Regression_pt: 0.0477 | Regression_sin_phi: 0.1081 | LR: 4.33e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3628s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank2 [2026-10-29 15:04:21,834] INFO : Step 29300/50000 rank2 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0426 | Regression_eta: 0.0245 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0424 | LR: 4.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0930s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank2 [2026-10-29 15:08:57,031] INFO : Step 29400/50000 rank2 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0491 | Regression_eta: 0.0621 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0827 | LR: 4.27e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2717s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank2 [2026-10-29 15:13:31,010] INFO : Step 29500/50000 rank2 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0637 | Regression_eta: 0.0305 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0566 | LR: 4.24e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4984s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank2 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank2 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0513 | Regression_eta: 0.0474 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0745 | LR: 4.22e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5924s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank2 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank2 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1221 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0653 | Regression_eta: 0.0490 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0793 | LR: 4.19e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6725s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank2 [2026-10-29 15:27:18,563] INFO : Step 29800/50000 rank2 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0630 | Regression_eta: 0.0554 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0594 | LR: 4.16e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6317s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank2 [2026-10-29 15:31:51,912] INFO : Step 29900/50000 rank2 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0521 | Regression_eta: 0.0439 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0538 | LR: 4.14e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5860s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank2 [2026-10-29 15:36:26,788] INFO : Step 30000/50000 rank2 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0575 | Regression_eta: 0.0374 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0684 | LR: 4.11e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4525s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank2 [2026-10-29 15:36:27,739] INFO : Running validation on rank2 +rank2 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 15:37:19,265] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 15:37:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 15:37:51,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 15:37:52,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 15:37:53,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 15:38:03,936] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:38:04,947] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 15:38:20,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 15:38:21,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 15:38:22,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 15:38:23,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 15:38:33,761] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:38:35,047] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 15:38:51,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 15:38:51,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 15:38:52,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 15:38:53,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 15:39:03,516] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank2 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0446 | Regression_eta: 0.0318 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0545 | LR: 4.08e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7103s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank2 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank2 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0624 | Regression_eta: 0.0296 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0674 | LR: 4.06e-04 | DataLoad Time: 0.0276s | Model Forward Time: 4.7621s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank2 [2026-10-29 15:53:38,867] INFO : Step 30300/50000 rank2 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0649 | Regression_eta: 0.0459 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0408 | LR: 4.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3364s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank2 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank2 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0573 | Regression_eta: 0.0335 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0561 | LR: 4.00e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4512s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank2 [2026-10-29 16:02:51,487] INFO : Step 30500/50000 rank2 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2680 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0674 | Regression_eta: 0.0568 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0759 | LR: 3.98e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.4576s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank2 [2026-10-29 16:07:32,050] INFO : Step 30600/50000 rank2 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0512 | Regression_eta: 0.0466 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0444 | LR: 3.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9379s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank2 [2026-10-29 16:12:18,780] INFO : Step 30700/50000 rank2 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1164 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0576 | Regression_eta: 0.0411 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0657 | LR: 3.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2831s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank2 [2026-10-29 16:17:03,636] INFO : Step 30800/50000 rank2 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0533 | Regression_eta: 0.0413 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0729 | LR: 3.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0642s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank2 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank2 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0481 | Regression_eta: 0.0350 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0483 | LR: 3.87e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6950s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank2 [2026-10-29 16:26:02,768] INFO : Step 31000/50000 rank2 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0460 | Regression_eta: 0.0263 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0525 | LR: 3.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0252s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank2 [2026-10-29 16:26:49,213] INFO : Running validation on rank2 +rank2 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:27:39,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:27:40,303] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 16:28:07,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:28:09,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:28:11,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:28:13,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:28:23,410] INFO : Time taken to make predictions on device 2 is: 0.30 min +rank2 [2026-10-29 16:28:24,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:28:24,724] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 16:28:40,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:28:41,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:28:42,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:28:43,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:28:53,616] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:28:54,191] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:28:54,676] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 16:29:10,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:29:11,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:29:12,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:29:12,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:29:22,955] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank2 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0473 | Regression_eta: 0.0323 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0412 | LR: 3.82e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.5512s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank2 [2026-10-29 16:39:13,624] INFO : Step 31200/50000 rank2 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0515 | Regression_eta: 0.0441 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0520 | LR: 3.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0724s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank2 [2026-10-29 16:43:57,175] INFO : Step 31300/50000 rank2 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0569 | Regression_eta: 0.0618 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0851 | LR: 3.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8847s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank2 [2026-10-29 16:48:30,666] INFO : Step 31400/50000 rank2 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1160 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0484 | Regression_eta: 0.0349 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0474 | LR: 3.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7840s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank2 [2026-10-29 16:53:07,751] INFO : Step 31500/50000 rank2 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0609 | Regression_eta: 0.0551 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0667 | LR: 3.71e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4921s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank2 [2026-10-29 16:57:38,607] INFO : Step 31600/50000 rank2 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0468 | Regression_eta: 0.0417 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0474 | LR: 3.69e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1676s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank2 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank2 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0498 | Regression_eta: 0.0434 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0484 | LR: 3.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1011s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank2 [2026-10-29 17:06:47,524] INFO : Step 31800/50000 rank2 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0491 | Regression_eta: 0.0386 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0651 | LR: 3.64e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2350s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank2 [2026-10-29 17:11:27,372] INFO : Step 31900/50000 rank2 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0509 | Regression_eta: 0.0326 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0494 | LR: 3.61e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5196s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank2 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank2 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1221 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0516 | Regression_eta: 0.0353 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0443 | LR: 3.58e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4962s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank2 [2026-10-29 17:16:12,935] INFO : Running validation on rank2 +rank2 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 17:17:03,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:17:04,359] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 17:17:32,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 17:17:33,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 17:17:34,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 17:17:35,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 17:17:45,272] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:17:46,318] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 17:18:02,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 17:18:03,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 17:18:04,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 17:18:04,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 17:18:14,969] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:18:15,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:18:16,464] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 17:18:31,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 17:18:32,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 17:18:33,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 17:18:34,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 17:18:44,073] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank2 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0626 | Regression_eta: 0.0305 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0696 | LR: 3.56e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9451s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank2 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank2 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1863 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0530 | Regression_eta: 0.0508 | Regression_pt: 0.0557 | Regression_sin_phi: 0.1044 | LR: 3.53e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.4533s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank2 [2026-10-29 17:33:21,418] INFO : Step 32300/50000 rank2 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0469 | Regression_eta: 0.0310 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0669 | LR: 3.51e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.9310s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank2 [2026-10-29 17:38:08,351] INFO : Step 32400/50000 rank2 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1218 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0563 | Regression_eta: 0.0387 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0424 | LR: 3.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8164s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank2 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank2 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0572 | Regression_eta: 0.0315 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0721 | LR: 3.46e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8574s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank2 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank2 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0448 | Regression_eta: 0.0333 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0527 | LR: 3.43e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6209s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank2 [2026-10-29 17:52:17,188] INFO : Step 32700/50000 rank2 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1311 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0610 | Regression_eta: 0.0351 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0896 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5864s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank2 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank2 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0517 | Regression_eta: 0.0253 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0415 | LR: 3.38e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank2 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank2 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0537 | Regression_eta: 0.0463 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0721 | LR: 3.36e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4246s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank2 [2026-10-29 18:06:22,611] INFO : Step 33000/50000 rank2 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0608 | Regression_eta: 0.0396 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0756 | LR: 3.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2118s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank2 [2026-10-29 18:06:23,517] INFO : Running validation on rank2 +rank2 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:07:15,766] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 18:07:43,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:07:44,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:07:45,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:07:46,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:07:57,000] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:07:58,401] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 18:08:14,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:08:15,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:08:16,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:08:17,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:08:27,168] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:08:28,344] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 18:08:43,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:08:44,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:08:45,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:08:46,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:08:56,157] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank2 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0380 | Regression_eta: 0.0428 | Regression_pt: 0.0385 | Regression_sin_phi: 0.0472 | LR: 3.31e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2616s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank2 [2026-10-29 18:18:33,981] INFO : Step 33200/50000 rank2 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0497 | Regression_eta: 0.0385 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0707 | LR: 3.28e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4340s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank2 [2026-10-29 18:23:15,420] INFO : Step 33300/50000 rank2 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0460 | Regression_eta: 0.0414 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0420 | LR: 3.26e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6015s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank2 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank2 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.0882 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0436 | Regression_eta: 0.0398 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0602 | LR: 3.23e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.2631s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank2 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank2 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0364 | Regression_eta: 0.0374 | Regression_pt: 0.0371 | Regression_sin_phi: 0.0646 | LR: 3.21e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3719s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank2 [2026-10-29 18:37:27,752] INFO : Step 33600/50000 rank2 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0444 | Regression_eta: 0.0461 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0640 | LR: 3.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3462s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank2 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank2 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0530 | Regression_eta: 0.0403 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0399 | LR: 3.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3186s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank2 [2026-10-29 18:47:04,985] INFO : Step 33800/50000 rank2 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0500 | Regression_eta: 0.0670 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0560 | LR: 3.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3939s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank2 [2026-10-29 18:51:52,247] INFO : Step 33900/50000 rank2 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0451 | Regression_eta: 0.0459 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0509 | LR: 3.11e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank2 [2026-10-29 18:56:35,807] INFO : Step 34000/50000 rank2 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0595 | Regression_eta: 0.0602 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0337 | LR: 3.09e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1631s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank2 [2026-10-29 18:56:36,807] INFO : Running validation on rank2 +rank2 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:57:29,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:57:30,356] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 18:57:54,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:57:55,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:57:57,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:57:58,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:58:08,103] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:58:09,212] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 18:58:25,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:58:26,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:58:27,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:58:28,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:58:38,300] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:58:39,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 18:58:54,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:58:55,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:58:56,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:58:57,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:59:07,344] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 19:04:16,497] INFO : Step 34100/50000 rank2 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0615 | Regression_eta: 0.0350 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0570 | LR: 3.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8497s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank2 [2026-10-29 19:09:14,637] INFO : Step 34200/50000 rank2 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0516 | Regression_eta: 0.0332 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0369 | LR: 3.04e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6994s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank2 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank2 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0444 | Regression_eta: 0.0329 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0578 | LR: 3.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3182s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank2 [2026-10-29 19:18:48,203] INFO : Step 34400/50000 rank2 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0645 | Regression_eta: 0.0314 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0460 | LR: 2.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9237s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank2 [2026-10-29 19:23:49,388] INFO : Step 34500/50000 rank2 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0632 | Regression_eta: 0.0359 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0442 | LR: 2.97e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2310s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank2 [2026-10-29 19:28:34,774] INFO : Step 34600/50000 rank2 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0402 | Regression_eta: 0.0470 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0472 | LR: 2.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4737s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank2 [2026-10-29 19:33:35,743] INFO : Step 34700/50000 rank2 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0407 | Regression_eta: 0.0279 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0348 | LR: 2.92e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1492s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank2 [2026-10-29 19:38:26,589] INFO : Step 34800/50000 rank2 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.0961 | Regression_cos_phi: 0.0320 | Regression_energy: 0.0360 | Regression_eta: 0.0230 | Regression_pt: 0.0375 | Regression_sin_phi: 0.0258 | LR: 2.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0549s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank2 [2026-10-29 19:43:15,574] INFO : Step 34900/50000 rank2 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0427 | Regression_eta: 0.0376 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0916 | LR: 2.88e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2769s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank2 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank2 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0513 | Regression_eta: 0.0327 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0699 | LR: 2.85e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3860s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank2 [2026-10-29 19:48:21,335] INFO : Running validation on rank2 +rank2 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 19:49:15,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:49:15,989] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 19:49:42,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 19:49:43,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 19:49:44,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 19:49:45,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 19:49:55,772] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:49:56,736] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 19:50:12,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 19:50:13,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 19:50:14,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 19:50:15,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 19:50:25,648] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:50:26,882] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 19:50:42,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 19:50:43,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 19:50:43,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 19:50:44,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 19:50:54,732] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 19:56:07,675] INFO : Step 35100/50000 rank2 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1122 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0573 | Regression_eta: 0.0264 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0673 | LR: 2.83e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4683s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank2 [2026-10-29 20:01:01,619] INFO : Step 35200/50000 rank2 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0458 | Regression_eta: 0.0273 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0387 | LR: 2.81e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6713s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank2 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank2 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0624 | Regression_eta: 0.0329 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0595 | LR: 2.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1450s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank2 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank2 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1205 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0466 | Regression_eta: 0.0243 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0471 | LR: 2.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4641s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank2 [2026-10-29 20:15:42,289] INFO : Step 35500/50000 rank2 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0598 | Regression_eta: 0.0466 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0532 | LR: 2.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3580s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank2 [2026-10-29 20:20:38,045] INFO : Step 35600/50000 rank2 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0620 | Regression_eta: 0.0503 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0518 | LR: 2.72e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1895s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank2 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank2 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0455 | Regression_eta: 0.0390 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0338 | LR: 2.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1727s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank2 [2026-10-29 20:30:22,090] INFO : Step 35800/50000 rank2 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1246 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0571 | Regression_eta: 0.0278 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0441 | LR: 2.68e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2784s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank2 [2026-10-29 20:35:20,188] INFO : Step 35900/50000 rank2 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0604 | Regression_eta: 0.0282 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0452 | LR: 2.65e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6687s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank2 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank2 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0534 | Regression_eta: 0.0358 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0525 | LR: 2.63e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2578s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank2 [2026-10-29 20:40:17,059] INFO : Running validation on rank2 +rank2 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 20:41:08,401] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:41:08,960] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 20:41:36,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 20:41:37,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 20:41:38,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 20:41:39,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 20:41:49,262] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:41:50,341] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 20:42:06,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 20:42:07,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 20:42:08,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 20:42:09,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 20:42:19,260] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:42:20,694] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 20:42:35,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 20:42:36,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 20:42:37,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 20:42:38,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 20:42:48,331] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 20:47:46,988] INFO : Step 36100/50000 rank2 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0582 | Regression_eta: 0.0260 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0477 | LR: 2.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4497s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank2 [2026-10-29 20:52:22,841] INFO : Step 36200/50000 rank2 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0497 | Regression_eta: 0.0293 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0573 | LR: 2.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank2 [2026-10-29 20:56:59,585] INFO : Step 36300/50000 rank2 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0520 | Regression_eta: 0.0262 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0563 | LR: 2.57e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2284s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank2 [2026-10-29 21:01:38,352] INFO : Step 36400/50000 rank2 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1218 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0450 | Regression_eta: 0.0260 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0437 | LR: 2.55e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8770s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank2 [2026-10-29 21:06:14,245] INFO : Step 36500/50000 rank2 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0527 | Regression_eta: 0.0489 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0519 | LR: 2.52e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4211s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank2 [2026-10-29 21:10:49,733] INFO : Step 36600/50000 rank2 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0479 | Regression_eta: 0.0295 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0436 | LR: 2.50e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2785s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank2 [2026-10-29 21:15:35,108] INFO : Step 36700/50000 rank2 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0402 | Regression_eta: 0.0382 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0477 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2693s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank2 [2026-10-29 21:20:28,003] INFO : Step 36800/50000 rank2 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0558 | Regression_eta: 0.0353 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0730 | LR: 2.46e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3984s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank2 [2026-10-29 21:25:23,681] INFO : Step 36900/50000 rank2 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0554 | Regression_eta: 0.0350 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0592 | LR: 2.44e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9347s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank2 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank2 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0436 | Regression_eta: 0.0245 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0503 | LR: 2.42e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9307s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank2 [2026-10-29 21:30:17,323] INFO : Running validation on rank2 +rank2 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:31:08,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:31:08,620] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 21:31:35,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:31:36,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:31:37,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:31:38,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:31:48,628] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:31:49,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 21:32:05,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:32:07,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:32:08,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:32:09,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:32:19,728] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:32:20,811] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 21:32:36,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:32:37,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:32:38,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:32:38,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:32:48,834] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 21:37:56,197] INFO : Step 37100/50000 rank2 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.0968 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0561 | Regression_eta: 0.0183 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0457 | LR: 2.40e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.6898s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank2 [2026-10-29 21:43:23,659] INFO : Step 37200/50000 rank2 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1244 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0485 | Regression_eta: 0.0252 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0525 | LR: 2.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6954s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank2 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank2 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0509 | Regression_eta: 0.0346 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0653 | LR: 2.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5583s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank2 [2026-10-29 21:52:56,909] INFO : Step 37400/50000 rank2 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0291 | Regression_energy: 0.0558 | Regression_eta: 0.0268 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0682 | LR: 2.34e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6110s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank2 [2026-10-29 21:57:43,877] INFO : Step 37500/50000 rank2 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0541 | Regression_eta: 0.0452 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0567 | LR: 2.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7102s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank2 [2026-10-29 22:02:29,840] INFO : Step 37600/50000 rank2 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0414 | Regression_eta: 0.0410 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0627 | LR: 2.30e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2468s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank2 [2026-10-29 22:07:05,609] INFO : Step 37700/50000 rank2 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0435 | Regression_eta: 0.0480 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0467 | LR: 2.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1552s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank2 [2026-10-29 22:11:38,774] INFO : Step 37800/50000 rank2 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0444 | Regression_eta: 0.0395 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0647 | LR: 2.26e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2570s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank2 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank2 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0393 | Regression_eta: 0.0318 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0464 | LR: 2.24e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1262s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank2 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank2 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0467 | Regression_eta: 0.0680 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0938 | LR: 2.22e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2889s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank2 [2026-10-29 22:20:57,876] INFO : Running validation on rank2 +rank2 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 22:21:48,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:21:48,922] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 22:22:15,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 22:22:16,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 22:22:17,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 22:22:18,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 22:22:28,377] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 22:22:28,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:22:29,428] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 22:22:45,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 22:22:46,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 22:22:47,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 22:22:48,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 22:22:58,548] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:22:59,393] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 22:23:14,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 22:23:15,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 22:23:16,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 22:23:17,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 22:23:27,535] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank2 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0592 | Regression_eta: 0.0291 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0386 | LR: 2.20e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2301s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank2 [2026-10-29 22:33:07,241] INFO : Step 38200/50000 rank2 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0479 | Regression_eta: 0.0432 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0644 | LR: 2.18e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7611s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank2 [2026-10-29 22:37:42,242] INFO : Step 38300/50000 rank2 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0543 | Regression_eta: 0.0443 | Regression_pt: 0.0563 | Regression_sin_phi: 0.1271 | LR: 2.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5928s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank2 [2026-10-29 22:42:17,608] INFO : Step 38400/50000 rank2 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1169 | Regression_cos_phi: 0.0369 | Regression_energy: 0.0552 | Regression_eta: 0.0291 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0410 | LR: 2.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4886s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank2 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank2 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0543 | Regression_eta: 0.0288 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0634 | LR: 2.12e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5219s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank2 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank2 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0442 | Regression_eta: 0.0299 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0574 | LR: 2.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7264s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank2 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank2 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0455 | Regression_eta: 0.0384 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0457 | LR: 2.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1562s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank2 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank2 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.0942 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0492 | Regression_eta: 0.0354 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0725 | LR: 2.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3982s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank2 [2026-10-29 23:07:29,432] INFO : Step 38900/50000 rank2 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0364 | Regression_eta: 0.0333 | Regression_pt: 0.0383 | Regression_sin_phi: 0.0482 | LR: 2.05e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8947s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank2 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank2 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0498 | Regression_eta: 0.0208 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0338 | LR: 2.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4703s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank2 [2026-10-29 23:12:38,866] INFO : Running validation on rank2 +rank2 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 23:13:33,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:13:34,230] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-29 23:14:06,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 23:14:07,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 23:14:08,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 23:14:09,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 23:14:19,738] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 23:14:20,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:14:20,797] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-29 23:14:36,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 23:14:37,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 23:14:38,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 23:14:39,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 23:14:49,764] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:14:50,898] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-29 23:15:06,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 23:15:07,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 23:15:08,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 23:15:08,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 23:15:18,754] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank2 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1004 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0347 | Regression_eta: 0.0404 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0825 | LR: 2.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9865s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank2 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank2 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0402 | Regression_eta: 0.0291 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0453 | LR: 2.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5910s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank2 [2026-10-29 23:29:56,591] INFO : Step 39300/50000 rank2 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0359 | Regression_eta: 0.0270 | Regression_pt: 0.0370 | Regression_sin_phi: 0.0584 | LR: 1.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6305s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank2 [2026-10-29 23:34:45,027] INFO : Step 39400/50000 rank2 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0465 | Regression_eta: 0.0405 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0684 | LR: 1.96e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.6709s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank2 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank2 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0444 | Regression_eta: 0.0309 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0484 | LR: 1.94e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4394s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank2 [2026-10-29 23:43:57,879] INFO : Step 39600/50000 rank2 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0422 | Regression_eta: 0.0306 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0454 | LR: 1.93e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4684s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank2 [2026-10-29 23:48:42,957] INFO : Step 39700/50000 rank2 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0438 | Regression_eta: 0.0271 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0431 | LR: 1.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4046s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank2 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank2 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1055 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0397 | Regression_eta: 0.0302 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0456 | LR: 1.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5378s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank2 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank2 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0410 | Regression_eta: 0.0256 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0409 | LR: 1.88e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7662s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank2 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank2 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0592 | Regression_eta: 0.0319 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0429 | LR: 1.86e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9280s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank2 [2026-10-30 00:05:56,736] INFO : Running validation on rank2 +rank2 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:06:50,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:06:51,353] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 00:07:17,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:07:18,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:07:19,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:07:20,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:07:30,918] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:07:31,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 00:07:47,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:07:48,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:07:49,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:07:50,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:08:00,742] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:08:01,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 00:08:17,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 00:08:18,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 00:08:19,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 00:08:20,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 00:08:30,492] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank2 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1053 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0477 | Regression_eta: 0.0326 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0500 | LR: 1.84e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6411s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank2 [2026-10-30 00:18:43,887] INFO : Step 40200/50000 rank2 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0454 | Regression_eta: 0.0296 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0604 | LR: 1.83e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4443s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank2 [2026-10-30 00:23:35,703] INFO : Step 40300/50000 rank2 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0441 | Regression_eta: 0.0292 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0743 | LR: 1.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5343s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank2 [2026-10-30 00:28:38,368] INFO : Step 40400/50000 rank2 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0453 | Regression_eta: 0.0308 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0423 | LR: 1.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2223s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank2 [2026-10-30 00:33:25,152] INFO : Step 40500/50000 rank2 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0356 | Regression_eta: 0.0290 | Regression_pt: 0.0379 | Regression_sin_phi: 0.0471 | LR: 1.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2273s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank2 [2026-10-30 00:38:07,831] INFO : Step 40600/50000 rank2 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0559 | Regression_eta: 0.0313 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0588 | LR: 1.76e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2967s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank2 [2026-10-30 00:42:54,583] INFO : Step 40700/50000 rank2 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1196 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0379 | Regression_eta: 0.0175 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0420 | LR: 1.75e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2540s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank2 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank2 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1007 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0393 | Regression_eta: 0.0205 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0416 | LR: 1.73e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8882s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank2 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank2 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0567 | Regression_eta: 0.0272 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0422 | LR: 1.72e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.7064s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank2 [2026-10-30 00:57:38,928] INFO : Step 41000/50000 rank2 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0263 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0511 | Regression_eta: 0.0370 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0468 | LR: 1.70e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3870s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank2 [2026-10-30 00:57:39,931] INFO : Running validation on rank2 +rank2 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:58:31,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:58:32,295] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 00:58:59,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:59:00,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:59:01,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:59:02,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:59:12,520] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:59:13,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:59:13,670] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 00:59:29,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:59:30,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:59:31,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:59:32,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:59:42,902] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:59:43,893] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 00:59:59,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:00:00,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:00:00,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:00:01,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:00:11,656] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 01:05:10,996] INFO : Step 41100/50000 rank2 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0298 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0479 | Regression_eta: 0.0265 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0403 | LR: 1.69e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7653s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank2 [2026-10-30 01:10:09,654] INFO : Step 41200/50000 rank2 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0258 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0510 | Regression_eta: 0.0317 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0440 | LR: 1.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6361s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank2 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank2 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0584 | Regression_eta: 0.0501 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0585 | LR: 1.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2625s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank2 [2026-10-30 01:19:55,521] INFO : Step 41400/50000 rank2 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1104 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0453 | Regression_eta: 0.0297 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0646 | LR: 1.64e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.6711s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank2 [2026-10-30 01:24:55,601] INFO : Step 41500/50000 rank2 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0377 | Regression_eta: 0.0344 | Regression_pt: 0.0394 | Regression_sin_phi: 0.0444 | LR: 1.63e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4501s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank2 [2026-10-30 01:29:39,759] INFO : Step 41600/50000 rank2 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0435 | Regression_eta: 0.0325 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0621 | LR: 1.61e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.0149s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank2 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank2 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0280 | Regression_energy: 0.0396 | Regression_eta: 0.0231 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0364 | LR: 1.60e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7049s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank2 [2026-10-30 01:39:10,939] INFO : Step 41800/50000 rank2 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1010 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0418 | Regression_eta: 0.0274 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0568 | LR: 1.58e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8798s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank2 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank2 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0460 | Regression_eta: 0.0233 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0435 | LR: 1.57e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2464s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank2 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank2 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0368 | Regression_eta: 0.0384 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0334 | LR: 1.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2695s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank2 [2026-10-30 01:49:03,869] INFO : Running validation on rank2 +rank2 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 01:49:57,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 01:50:25,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 01:50:26,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 01:50:27,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 01:50:28,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 01:50:38,268] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:50:39,486] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 01:50:55,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 01:50:56,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 01:50:57,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 01:50:58,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 01:51:08,680] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:51:10,101] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 01:51:25,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:51:26,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:51:27,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:51:27,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:51:37,788] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank2 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0439 | Regression_eta: 0.0277 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0457 | LR: 1.54e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0036s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank2 [2026-10-30 02:01:47,468] INFO : Step 42200/50000 rank2 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0372 | Regression_eta: 0.0359 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0477 | LR: 1.53e-04 | DataLoad Time: 0.0156s | Model Forward Time: 7.2731s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank2 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank2 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-30 02:06:41,298] INFO : Step 42300/50000 rank2 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0484 | Regression_eta: 0.0307 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0539 | LR: 1.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8862s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank2 [2026-10-30 02:11:46,427] INFO : Step 42400/50000 rank2 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1009 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0616 | Regression_eta: 0.0399 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0524 | LR: 1.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1598s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank2 [2026-10-30 02:16:38,978] INFO : Step 42500/50000 rank2 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1568 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0669 | Regression_eta: 0.0288 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0542 | LR: 1.49e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank2 [2026-10-30 02:21:27,109] INFO : Step 42600/50000 rank2 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0458 | Regression_eta: 0.0317 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0434 | LR: 1.48e-04 | DataLoad Time: 0.0185s | Model Forward Time: 6.3557s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank2 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank2 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0625 | Regression_eta: 0.0474 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0464 | LR: 1.47e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6229s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank2 [2026-10-30 02:31:38,796] INFO : Step 42800/50000 rank2 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1037 | Regression_cos_phi: 0.0348 | Regression_energy: 0.0487 | Regression_eta: 0.0163 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0235 | LR: 1.45e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6408s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank2 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank2 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0455 | Regression_eta: 0.0419 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0777 | LR: 1.44e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8530s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank2 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank2 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0271 | Regression_energy: 0.0403 | Regression_eta: 0.0317 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0392 | LR: 1.43e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank2 [2026-10-30 02:41:14,879] INFO : Running validation on rank2 +rank2 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 02:42:10,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 02:42:38,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 02:42:39,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 02:42:40,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 02:42:41,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 02:42:51,372] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 02:42:51,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:42:52,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 02:43:08,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 02:43:09,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 02:43:10,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 02:43:11,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 02:43:21,917] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 02:43:38,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 02:43:39,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 02:43:40,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 02:43:41,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 02:43:51,427] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 02:48:51,376] INFO : Step 43100/50000 rank2 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.0993 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0502 | Regression_eta: 0.0239 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0304 | LR: 1.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9462s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank2 [2026-10-30 02:53:29,171] INFO : Step 43200/50000 rank2 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0497 | Regression_eta: 0.0318 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0425 | LR: 1.40e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5632s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank2 [2026-10-30 02:58:07,501] INFO : Step 43300/50000 rank2 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0580 | Regression_eta: 0.0343 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0506 | LR: 1.39e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.0497s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank2 [2026-10-30 03:02:42,808] INFO : Step 43400/50000 rank2 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0472 | Regression_eta: 0.0208 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0477 | LR: 1.38e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4745s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank2 [2026-10-30 03:07:16,794] INFO : Step 43500/50000 rank2 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1231 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0426 | Regression_eta: 0.0265 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0530 | LR: 1.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9332s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank2 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank2 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0367 | Regression_eta: 0.0235 | Regression_pt: 0.0385 | Regression_sin_phi: 0.0503 | LR: 1.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8416s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank2 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank2 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0448 | Regression_eta: 0.0569 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0610 | LR: 1.35e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1354s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank2 [2026-10-30 03:21:10,514] INFO : Step 43800/50000 rank2 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1243 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0473 | Regression_eta: 0.0251 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0392 | LR: 1.34e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2529s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank2 [2026-10-30 03:26:01,000] INFO : Step 43900/50000 rank2 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0502 | Regression_eta: 0.0412 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0865 | LR: 1.33e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9693s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank2 [2026-10-30 03:30:36,933] INFO : Step 44000/50000 rank2 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0410 | Regression_eta: 0.0386 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0458 | LR: 1.32e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8180s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank2 [2026-10-30 03:30:37,938] INFO : Running validation on rank2 +rank2 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 03:31:28,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:31:29,165] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 03:31:57,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 03:31:58,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 03:31:59,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 03:32:00,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 03:32:10,664] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:32:11,904] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 03:32:27,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 03:32:28,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 03:32:29,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 03:32:30,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 03:32:40,689] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:32:41,806] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 03:32:57,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 03:32:58,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 03:32:58,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 03:32:59,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 03:33:09,732] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 03:38:15,294] INFO : Step 44100/50000 rank2 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0364 | Regression_eta: 0.0193 | Regression_pt: 0.0381 | Regression_sin_phi: 0.0285 | LR: 1.31e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7481s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank2 [2026-10-30 03:43:04,521] INFO : Step 44200/50000 rank2 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0420 | Regression_eta: 0.0329 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0787 | LR: 1.30e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4214s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank2 [2026-10-30 03:47:46,464] INFO : Step 44300/50000 rank2 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1424 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0421 | Regression_eta: 0.0328 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0547 | LR: 1.29e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4327s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank2 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank2 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0550 | Regression_eta: 0.0258 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0376 | LR: 1.28e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3358s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank2 [2026-10-30 03:57:11,971] INFO : Step 44500/50000 rank2 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1104 | Regression_cos_phi: 0.0266 | Regression_energy: 0.0341 | Regression_eta: 0.0227 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0366 | LR: 1.27e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3160s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank2 [2026-10-30 04:01:48,196] INFO : Step 44600/50000 rank2 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0536 | Regression_eta: 0.0467 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0623 | LR: 1.26e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5952s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank2 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank2 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0206 | Classification_binary: 0.1030 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0472 | Regression_eta: 0.0211 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0438 | LR: 1.25e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.6212s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank2 [2026-10-30 04:11:07,220] INFO : Step 44800/50000 rank2 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0258 | Regression_energy: 0.0402 | Regression_eta: 0.0234 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0312 | LR: 1.24e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5867s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank2 [2026-10-30 04:15:54,537] INFO : Step 44900/50000 rank2 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0457 | Regression_eta: 0.0201 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0262 | LR: 1.23e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6729s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank2 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank2 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0428 | Regression_eta: 0.0320 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0396 | LR: 1.22e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7910s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank2 [2026-10-30 04:20:37,109] INFO : Running validation on rank2 +rank2 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 04:21:27,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:21:28,557] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 04:21:56,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 04:21:57,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 04:21:58,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 04:22:00,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 04:22:10,343] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:22:10,892] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 04:22:27,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 04:22:28,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 04:22:29,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 04:22:30,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 04:22:40,190] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:22:41,057] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 04:22:56,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 04:22:57,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 04:22:58,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 04:22:58,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 04:23:08,948] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank2 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0509 | Regression_eta: 0.0395 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0556 | LR: 1.21e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2102s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank2 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank2 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1276 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0415 | Regression_eta: 0.0400 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0434 | LR: 1.20e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4468s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank2 [2026-10-30 04:37:52,084] INFO : Step 45300/50000 rank2 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0585 | Regression_eta: 0.0366 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0396 | LR: 1.19e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8142s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank2 [2026-10-30 04:42:38,427] INFO : Step 45400/50000 rank2 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0474 | Regression_eta: 0.0423 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0546 | LR: 1.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4021s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank2 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank2 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0553 | Regression_eta: 0.0319 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0553 | LR: 1.18e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3863s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank2 [2026-10-30 04:52:16,190] INFO : Step 45600/50000 rank2 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0412 | Regression_eta: 0.0230 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0554 | LR: 1.17e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.4809s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank2 [2026-10-30 04:57:07,845] INFO : Step 45700/50000 rank2 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.0994 | Regression_cos_phi: 0.0284 | Regression_energy: 0.0378 | Regression_eta: 0.0182 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0320 | LR: 1.16e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.4881s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank2 [2026-10-30 05:01:56,082] INFO : Step 45800/50000 rank2 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1049 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0437 | Regression_eta: 0.0207 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0420 | LR: 1.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6477s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank2 [2026-10-30 05:06:42,062] INFO : Step 45900/50000 rank2 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0256 | Regression_energy: 0.0316 | Regression_eta: 0.0257 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0428 | LR: 1.15e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2870s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank2 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank2 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0361 | Regression_eta: 0.0305 | Regression_pt: 0.0377 | Regression_sin_phi: 0.0317 | LR: 1.14e-04 | DataLoad Time: 0.0210s | Model Forward Time: 6.4340s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank2 [2026-10-30 05:11:37,038] INFO : Running validation on rank2 +rank2 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 05:12:27,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:12:27,994] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 05:12:53,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 05:12:54,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 05:12:55,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 05:12:56,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 05:13:06,252] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 05:13:23,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 05:13:24,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 05:13:25,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 05:13:26,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 05:13:36,052] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 05:13:36,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 05:13:52,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 05:13:53,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 05:13:54,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 05:13:54,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 05:14:04,957] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 05:19:00,537] INFO : Step 46100/50000 rank2 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0482 | Regression_eta: 0.0296 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0393 | LR: 1.13e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5902s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank2 [2026-10-30 05:23:34,446] INFO : Step 46200/50000 rank2 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0541 | Regression_eta: 0.0265 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0316 | LR: 1.13e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7923s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank2 [2026-10-30 05:28:08,608] INFO : Step 46300/50000 rank2 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0623 | Regression_eta: 0.0336 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0469 | LR: 1.12e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9779s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank2 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank2 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0615 | Regression_eta: 0.0396 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0478 | LR: 1.11e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7881s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank2 [2026-10-30 05:37:21,831] INFO : Step 46500/50000 rank2 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.0918 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0445 | Regression_eta: 0.0261 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0557 | LR: 1.11e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8410s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank2 [2026-10-30 05:41:57,343] INFO : Step 46600/50000 rank2 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0383 | Regression_eta: 0.0252 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0314 | LR: 1.10e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7840s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank2 [2026-10-30 05:46:32,411] INFO : Step 46700/50000 rank2 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0454 | Regression_eta: 0.0195 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0518 | LR: 1.10e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6534s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank2 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank2 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0473 | Regression_eta: 0.0241 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0448 | LR: 1.09e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7028s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank2 [2026-10-30 05:55:41,597] INFO : Step 46900/50000 rank2 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0444 | Regression_eta: 0.0387 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0502 | LR: 1.09e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7521s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank2 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank2 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0473 | Regression_eta: 0.0350 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0716 | LR: 1.08e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8061s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank2 [2026-10-30 06:00:21,750] INFO : Running validation on rank2 +rank2 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 06:01:12,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 06:01:37,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 06:01:38,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 06:01:39,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 06:01:40,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 06:01:50,299] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:01:51,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 06:02:08,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 06:02:09,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 06:02:10,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 06:02:11,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 06:02:21,216] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:02:21,871] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 06:02:37,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 06:02:38,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 06:02:39,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 06:02:39,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 06:02:49,999] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 06:07:51,195] INFO : Step 47100/50000 rank2 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0389 | Regression_eta: 0.0337 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0450 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6254s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank2 [2026-10-30 06:12:29,637] INFO : Step 47200/50000 rank2 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0381 | Regression_eta: 0.0262 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0417 | LR: 1.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9899s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank2 [2026-10-30 06:17:04,150] INFO : Step 47300/50000 rank2 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0547 | Regression_eta: 0.0403 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0531 | LR: 1.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0314s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank2 [2026-10-30 06:21:38,449] INFO : Step 47400/50000 rank2 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0462 | Regression_eta: 0.0332 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0567 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank2 [2026-10-30 06:26:15,801] INFO : Step 47500/50000 rank2 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1003 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0531 | Regression_eta: 0.0261 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0378 | LR: 1.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6773s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank2 [2026-10-30 06:30:50,119] INFO : Step 47600/50000 rank2 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0443 | Regression_eta: 0.0358 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0601 | LR: 1.05e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.9496s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank2 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank2 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0414 | Regression_eta: 0.0179 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0388 | LR: 1.05e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9101s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank2 [2026-10-30 06:40:00,409] INFO : Step 47800/50000 rank2 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1158 | Regression_cos_phi: 0.0297 | Regression_energy: 0.0464 | Regression_eta: 0.0209 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0433 | LR: 1.04e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7815s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank2 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank2 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0243 | Classification_binary: 0.1068 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0379 | Regression_eta: 0.0221 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0390 | LR: 1.04e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8498s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank2 [2026-10-30 06:49:09,637] INFO : Step 48000/50000 rank2 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0415 | Regression_eta: 0.0279 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0340 | LR: 1.04e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8830s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank2 [2026-10-30 06:49:10,727] INFO : Running validation on rank2 +rank2 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 06:50:01,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 06:50:27,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 06:50:28,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 06:50:29,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 06:50:30,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 06:50:40,461] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:50:41,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:50:41,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 06:50:57,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 06:50:58,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 06:50:59,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 06:51:00,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 06:51:10,913] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:51:11,435] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 06:51:27,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 06:51:28,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 06:51:28,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 06:51:29,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 06:51:39,700] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank2 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1062 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0437 | Regression_eta: 0.0412 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0298 | LR: 1.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6274s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank2 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank2 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0392 | Regression_eta: 0.0354 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0948 | LR: 1.03e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.3150s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank2 [2026-10-30 07:05:53,466] INFO : Step 48300/50000 rank2 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0414 | Regression_eta: 0.0239 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0411 | LR: 1.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1176s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank2 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank2 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0494 | Regression_eta: 0.0494 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0713 | LR: 1.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8693s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank2 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank2 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0435 | Regression_eta: 0.0292 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0439 | LR: 1.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.9615s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank2 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank2 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0444 | Regression_eta: 0.0366 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0449 | LR: 1.02e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8070s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank2 [2026-10-30 07:24:30,058] INFO : Step 48700/50000 rank2 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0397 | Regression_eta: 0.0229 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0421 | LR: 1.02e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9968s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank2 [2026-10-30 07:29:10,218] INFO : Step 48800/50000 rank2 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1153 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0503 | Regression_eta: 0.0312 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0400 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9230s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank2 [2026-10-30 07:33:48,297] INFO : Step 48900/50000 rank2 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1290 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0437 | Regression_eta: 0.0372 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0615 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7892s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank2 [2026-10-30 07:38:22,688] INFO : Step 49000/50000 rank2 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0269 | Regression_energy: 0.0490 | Regression_eta: 0.0294 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0377 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8884s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank2 [2026-10-30 07:38:23,699] INFO : Running validation on rank2 +rank2 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:39:14,628] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 07:39:39,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 07:39:40,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 07:39:41,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 07:39:42,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 07:39:52,593] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 07:39:53,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:39:53,776] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 07:40:09,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 07:40:10,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 07:40:11,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 07:40:12,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 07:40:22,543] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:40:24,012] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 07:40:39,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 07:40:40,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 07:40:41,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 07:40:41,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 07:40:51,997] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank2 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1250 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0938 | Regression_eta: 0.0409 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0512 | LR: 1.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6549s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank2 [2026-10-30 07:50:30,114] INFO : Step 49200/50000 rank2 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0168 | Regression_energy: 0.0367 | Regression_eta: 0.0216 | Regression_pt: 0.0363 | Regression_sin_phi: 0.0485 | LR: 1.01e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5159s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank2 [2026-10-30 07:55:07,371] INFO : Step 49300/50000 rank2 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0415 | Regression_eta: 0.0688 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0538 | LR: 1.00e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8691s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank2 [2026-10-30 07:59:43,351] INFO : Step 49400/50000 rank2 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1266 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0519 | Regression_eta: 0.0282 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0501 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8896s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank2 [2026-10-30 08:04:24,759] INFO : Step 49500/50000 rank2 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1153 | Regression_cos_phi: 0.0343 | Regression_energy: 0.0447 | Regression_eta: 0.0227 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0357 | LR: 1.00e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3364s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank2 [2026-10-30 08:08:58,924] INFO : Step 49600/50000 rank2 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0339 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0388 | Regression_eta: 0.0274 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0316 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8769s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank2 [2026-10-30 08:13:36,933] INFO : Step 49700/50000 rank2 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0333 | Regression_energy: 0.0364 | Regression_eta: 0.0307 | Regression_pt: 0.0368 | Regression_sin_phi: 0.0477 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7054s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank2 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank2 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0439 | Regression_eta: 0.0478 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0527 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5264s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank2 [2026-10-30 08:22:53,416] INFO : Step 49900/50000 rank2 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0506 | Regression_eta: 0.0244 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0383 | LR: 1.00e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.2011s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank2 [2026-10-30 08:27:37,250] INFO : Step 50000/50000 rank2 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0353 | Regression_eta: 0.0223 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0420 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3464s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank2 [2026-10-30 08:27:38,187] INFO : Running validation on rank2 +rank2 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 08:28:29,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 08:28:29,585] INFO : Running predictions on clic_edm_ttbar_hits +rank2 [2026-10-30 08:29:00,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 08:29:01,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 08:29:02,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 08:29:03,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 08:29:13,305] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 08:29:13,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 08:29:14,404] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank2 [2026-10-30 08:29:30,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 08:29:32,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 08:29:33,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 08:29:34,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 08:29:44,908] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 08:29:46,125] INFO : Running predictions on clic_edm_qq_hits +rank2 [2026-10-30 08:30:01,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 08:30:02,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 08:30:03,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 08:30:03,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 08:30:13,841] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 08:30:15,198] INFO : Training completed. Total time on device 2: 1059.926min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.3 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..6356a9b183ab6f0d8103956de049ac4e0ba54cb8 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.3 @@ -0,0 +1,2654 @@ +rank3 [2026-10-28 11:12:32,425] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-28 11:12:32,426] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank3 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank3 [2026-10-28 11:12:57,209] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-28 11:12:57,209] INFO : Initializing backbone layer 0 +rank3 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank3 [2026-10-28 11:12:57,254] INFO : Initializing backbone layer 2 +rank3 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 3 +rank3 [2026-10-28 11:12:57,272] INFO : Initializing backbone layer 4 +rank3 [2026-10-28 11:12:57,280] INFO : Initializing backbone layer 5 +rank3 [2026-10-28 11:12:57,289] INFO : Backbone initialization took 0.08s +rank3 [2026-10-28 11:12:57,289] INFO : backbone parameters: 7901376 +rank3 [2026-10-28 11:12:57,289] INFO : Initializing output DNNs +rank3 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank3 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank3 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank3 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank3 [2026-10-28 11:12:57,328] INFO : classification_norm parameters: 0 +rank3 [2026-10-28 11:12:57,328] INFO : regression_norm parameters: 0 +rank3 [2026-10-28 11:12:57,328] INFO : classification_query parameters: 0 +rank3 [2026-10-28 11:12:57,328] INFO : regression_query parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : classification_readout parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : regression_readout parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : output_mode=set +rank3 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank3 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank3 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank3 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank3 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=3 +rank3 [2026-10-28 11:12:57,368] INFO : Moved model to device rank=3 +rank3 [2026-10-28 11:12:57,370] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-28 11:12:57,370] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-28 11:12:57,856] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank3 [2026-10-28 11:12:57,856] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-28 11:12:57,857] INFO : Trainable parameters: 12786964 +rank3 [2026-10-28 11:12:57,857] INFO : Non-trainable parameters: 23268 +rank3 [2026-10-28 11:12:57,857] INFO : Total parameters: 12810232 +rank3 [2026-10-28 11:12:57,860] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-28 11:12:57,861] INFO : Getting dataloaders +rank3 [2026-10-28 11:12:57,861] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:03,731] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:04,758] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-28 11:13:06,003] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,293] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,540] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,799] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-28 11:13:06,799] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-28 11:13:21,086] INFO : Got dataloaders +rank3 [2026-10-28 11:13:21,086] INFO : loader split=train rank=3 len=21108 +rank3 [2026-10-28 11:13:21,086] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-28 11:13:21,086] INFO : Starting training from step 1 to 50000 on rank 3 of 8 +rank3 [2026-10-28 11:13:21,087] INFO : Creating train_iterator +rank3 [2026-10-28 11:13:21,087] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-28 11:13:35,778] INFO : Created train_iterator +rank3 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank3 [2026-10-28 11:18:16,511] INFO : Step 100/50000 rank3 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7664 | Classification_binary: 0.4431 | Regression_cos_phi: 0.0049 | Regression_energy: 0.7818 | Regression_eta: 0.0137 | Regression_pt: 0.7181 | Regression_sin_phi: 0.0067 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 3.6176s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank3 [2026-10-28 11:22:47,963] INFO : Step 200/50000 rank3 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3622 | Classification_binary: 0.5087 | Regression_cos_phi: 0.4000 | Regression_energy: 0.3675 | Regression_eta: 0.3335 | Regression_pt: 0.3631 | Regression_sin_phi: 0.3142 | LR: 1.00e-03 | DataLoad Time: 0.0148s | Model Forward Time: 4.4396s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank3 [2026-10-28 11:27:17,891] INFO : Step 300/50000 rank3 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2546 | Classification_binary: 0.4913 | Regression_cos_phi: 0.3353 | Regression_energy: 0.2904 | Regression_eta: 0.2232 | Regression_pt: 0.2975 | Regression_sin_phi: 0.2638 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 4.3672s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank3 [2026-10-28 11:31:48,895] INFO : Step 400/50000 rank3 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2642 | Classification_binary: 0.4066 | Regression_cos_phi: 0.2725 | Regression_energy: 0.3154 | Regression_eta: 0.1791 | Regression_pt: 0.3116 | Regression_sin_phi: 0.1977 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.2446s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank3 [2026-10-28 11:36:20,165] INFO : Step 500/50000 rank3 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2955 | Classification_binary: 0.4736 | Regression_cos_phi: 0.3020 | Regression_energy: 0.3666 | Regression_eta: 0.2400 | Regression_pt: 0.3808 | Regression_sin_phi: 0.3050 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 4.4697s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank3 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank3 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2262 | Classification_binary: 0.4685 | Regression_cos_phi: 0.2894 | Regression_energy: 0.2554 | Regression_eta: 0.1838 | Regression_pt: 0.2635 | Regression_sin_phi: 0.2278 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.5135s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank3 [2026-10-28 11:45:32,573] INFO : Step 700/50000 rank3 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1942 | Classification_binary: 0.3961 | Regression_cos_phi: 0.2524 | Regression_energy: 0.2493 | Regression_eta: 0.1734 | Regression_pt: 0.2568 | Regression_sin_phi: 0.2413 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 3.7408s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank3 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank3 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2508 | Classification_binary: 0.4612 | Regression_cos_phi: 0.3403 | Regression_energy: 0.2940 | Regression_eta: 0.2449 | Regression_pt: 0.3070 | Regression_sin_phi: 0.3352 | LR: 9.99e-04 | DataLoad Time: 0.0227s | Model Forward Time: 5.2804s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank3 [2026-10-28 11:54:58,536] INFO : Step 900/50000 rank3 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2590 | Classification_binary: 0.5141 | Regression_cos_phi: 0.3421 | Regression_energy: 0.2978 | Regression_eta: 0.2522 | Regression_pt: 0.3007 | Regression_sin_phi: 0.3173 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4566s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank3 [2026-10-28 11:59:55,232] INFO : Step 1000/50000 rank3 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2069 | Classification_binary: 0.4156 | Regression_cos_phi: 0.1733 | Regression_energy: 0.2801 | Regression_eta: 0.1450 | Regression_pt: 0.2893 | Regression_sin_phi: 0.1707 | LR: 9.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.7403s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank3 [2026-10-28 11:59:56,332] INFO : Running validation on rank3 +rank3 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 12:00:48,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:00:49,300] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 12:01:16,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:01:17,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:01:18,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:01:19,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:01:29,958] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:01:30,838] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 12:01:46,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:01:47,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:01:48,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:01:49,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:01:59,328] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 12:02:00,465] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:02:01,007] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 12:02:16,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:02:17,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:02:18,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:02:19,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:02:29,309] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:07:33,551] INFO : Step 1100/50000 rank3 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1786 | Classification_binary: 0.4170 | Regression_cos_phi: 0.3021 | Regression_energy: 0.2751 | Regression_eta: 0.1782 | Regression_pt: 0.2847 | Regression_sin_phi: 0.2329 | LR: 9.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8137s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank3 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank3 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2231 | Classification_binary: 0.4927 | Regression_cos_phi: 0.3253 | Regression_energy: 0.2573 | Regression_eta: 0.2154 | Regression_pt: 0.2677 | Regression_sin_phi: 0.3100 | LR: 9.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4366s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank3 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank3 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1713 | Classification_binary: 0.3938 | Regression_cos_phi: 0.2493 | Regression_energy: 0.2668 | Regression_eta: 0.2244 | Regression_pt: 0.2790 | Regression_sin_phi: 0.2238 | LR: 9.98e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2259s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank3 [2026-10-28 12:21:55,310] INFO : Step 1400/50000 rank3 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2141 | Classification_binary: 0.3901 | Regression_cos_phi: 0.3076 | Regression_energy: 0.2456 | Regression_eta: 0.1942 | Regression_pt: 0.2465 | Regression_sin_phi: 0.2198 | LR: 9.98e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5563s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank3 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank3 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2302 | Classification_binary: 0.4115 | Regression_cos_phi: 0.2307 | Regression_energy: 0.2510 | Regression_eta: 0.1661 | Regression_pt: 0.2641 | Regression_sin_phi: 0.2423 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9960s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank3 [2026-10-28 12:31:13,938] INFO : Step 1600/50000 rank3 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 0.4263 | Regression_cos_phi: 0.2516 | Regression_energy: 0.2471 | Regression_eta: 0.1753 | Regression_pt: 0.2545 | Regression_sin_phi: 0.2115 | LR: 9.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8058s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank3 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank3 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1823 | Classification_binary: 0.3504 | Regression_cos_phi: 0.2668 | Regression_energy: 0.2176 | Regression_eta: 0.1615 | Regression_pt: 0.2201 | Regression_sin_phi: 0.2514 | LR: 9.97e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank3 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank3 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1999 | Classification_binary: 0.3994 | Regression_cos_phi: 0.2520 | Regression_energy: 0.2311 | Regression_eta: 0.1815 | Regression_pt: 0.2413 | Regression_sin_phi: 0.2263 | LR: 9.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6774s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank3 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank3 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1812 | Classification_binary: 0.4446 | Regression_cos_phi: 0.2502 | Regression_energy: 0.2267 | Regression_eta: 0.1743 | Regression_pt: 0.2382 | Regression_sin_phi: 0.2528 | LR: 9.97e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4313s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank3 [2026-10-28 12:50:22,602] INFO : Step 2000/50000 rank3 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1205 | Classification_binary: 0.3652 | Regression_cos_phi: 0.2364 | Regression_energy: 0.2018 | Regression_eta: 0.1514 | Regression_pt: 0.2087 | Regression_sin_phi: 0.2564 | LR: 9.96e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.9897s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank3 [2026-10-28 12:50:23,688] INFO : Running validation on rank3 +rank3 [2026-10-28 12:50:23,688] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 12:50:23,688] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 12:51:19,820] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:51:20,638] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 12:51:47,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:51:48,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:51:49,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:51:51,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:52:01,229] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 12:52:17,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:52:18,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:52:19,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:52:20,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:52:30,745] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:52:31,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:52:32,091] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 12:52:48,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:52:49,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:52:50,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:52:51,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:53:01,257] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank3 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1602 | Classification_binary: 0.3421 | Regression_cos_phi: 0.1866 | Regression_energy: 0.2350 | Regression_eta: 0.1392 | Regression_pt: 0.2397 | Regression_sin_phi: 0.1636 | LR: 9.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6511s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank3 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank3 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1651 | Classification_binary: 0.3872 | Regression_cos_phi: 0.2304 | Regression_energy: 0.2544 | Regression_eta: 0.1486 | Regression_pt: 0.2656 | Regression_sin_phi: 0.1897 | LR: 9.96e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.4349s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank3 [2026-10-28 13:08:12,324] INFO : Step 2300/50000 rank3 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2332 | Classification_binary: 0.4164 | Regression_cos_phi: 0.2349 | Regression_energy: 0.2521 | Regression_eta: 0.1859 | Regression_pt: 0.2668 | Regression_sin_phi: 0.2378 | LR: 9.95e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1085s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank3 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank3 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.2819 | Regression_cos_phi: 0.2588 | Regression_energy: 0.2387 | Regression_eta: 0.2068 | Regression_pt: 0.2477 | Regression_sin_phi: 0.2787 | LR: 9.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2184s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank3 [2026-10-28 13:18:06,874] INFO : Step 2500/50000 rank3 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1994 | Classification_binary: 0.3866 | Regression_cos_phi: 0.2631 | Regression_energy: 0.2243 | Regression_eta: 0.1741 | Regression_pt: 0.2308 | Regression_sin_phi: 0.2566 | LR: 9.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.0075s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank3 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank3 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.3949 | Regression_cos_phi: 0.2240 | Regression_energy: 0.2380 | Regression_eta: 0.1446 | Regression_pt: 0.2487 | Regression_sin_phi: 0.1638 | LR: 9.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3782s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank3 [2026-10-28 13:27:39,806] INFO : Step 2700/50000 rank3 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1345 | Classification_binary: 0.3421 | Regression_cos_phi: 0.1799 | Regression_energy: 0.1973 | Regression_eta: 0.1509 | Regression_pt: 0.2019 | Regression_sin_phi: 0.1889 | LR: 9.94e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5217s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank3 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank3 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1395 | Classification_binary: 0.3192 | Regression_cos_phi: 0.2243 | Regression_energy: 0.2472 | Regression_eta: 0.1328 | Regression_pt: 0.2534 | Regression_sin_phi: 0.1757 | LR: 9.93e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2921s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank3 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank3 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1680 | Classification_binary: 0.4448 | Regression_cos_phi: 0.2351 | Regression_energy: 0.2426 | Regression_eta: 0.1736 | Regression_pt: 0.2519 | Regression_sin_phi: 0.2390 | LR: 9.93e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5104s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank3 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank3 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1421 | Classification_binary: 0.3252 | Regression_cos_phi: 0.1826 | Regression_energy: 0.1905 | Regression_eta: 0.1377 | Regression_pt: 0.2000 | Regression_sin_phi: 0.1514 | LR: 9.92e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5886s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank3 [2026-10-28 13:42:04,672] INFO : Running validation on rank3 +rank3 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:42:57,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:42:58,469] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 13:43:25,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:43:26,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:43:27,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:43:28,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:43:38,258] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:43:39,345] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 13:43:55,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:43:56,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:43:57,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:43:58,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:44:08,570] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:44:09,103] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 13:44:24,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:44:25,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:44:26,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:44:27,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:44:37,225] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 13:49:29,792] INFO : Step 3100/50000 rank3 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1300 | Classification_binary: 0.3077 | Regression_cos_phi: 0.1679 | Regression_energy: 0.1903 | Regression_eta: 0.1429 | Regression_pt: 0.1975 | Regression_sin_phi: 0.1888 | LR: 9.91e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8431s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank3 [2026-10-28 13:54:22,657] INFO : Step 3200/50000 rank3 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1889 | Classification_binary: 0.4185 | Regression_cos_phi: 0.2894 | Regression_energy: 0.2098 | Regression_eta: 0.1924 | Regression_pt: 0.2204 | Regression_sin_phi: 0.2693 | LR: 9.91e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0534s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank3 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank3 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3778 | Regression_cos_phi: 0.1705 | Regression_energy: 0.1859 | Regression_eta: 0.1414 | Regression_pt: 0.1911 | Regression_sin_phi: 0.2259 | LR: 9.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7407s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank3 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank3 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1346 | Classification_binary: 0.3114 | Regression_cos_phi: 0.1868 | Regression_energy: 0.1975 | Regression_eta: 0.2020 | Regression_pt: 0.1992 | Regression_sin_phi: 0.2615 | LR: 9.90e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9447s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank3 [2026-10-28 14:08:16,394] INFO : Step 3500/50000 rank3 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1458 | Classification_binary: 0.3384 | Regression_cos_phi: 0.1857 | Regression_energy: 0.1852 | Regression_eta: 0.1167 | Regression_pt: 0.1958 | Regression_sin_phi: 0.1492 | LR: 9.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7452s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank3 [2026-10-28 14:13:02,151] INFO : Step 3600/50000 rank3 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1486 | Classification_binary: 0.4062 | Regression_cos_phi: 0.2278 | Regression_energy: 0.2010 | Regression_eta: 0.1369 | Regression_pt: 0.2089 | Regression_sin_phi: 0.1948 | LR: 9.89e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8788s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank3 [2026-10-28 14:17:42,401] INFO : Step 3700/50000 rank3 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1293 | Classification_binary: 0.2832 | Regression_cos_phi: 0.1809 | Regression_energy: 0.2058 | Regression_eta: 0.1465 | Regression_pt: 0.2100 | Regression_sin_phi: 0.1631 | LR: 9.88e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.2570s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank3 [2026-10-28 14:22:44,343] INFO : Step 3800/50000 rank3 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1457 | Classification_binary: 0.3767 | Regression_cos_phi: 0.2541 | Regression_energy: 0.2070 | Regression_eta: 0.1794 | Regression_pt: 0.2150 | Regression_sin_phi: 0.2041 | LR: 9.87e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank3 [2026-10-28 14:27:33,638] INFO : Step 3900/50000 rank3 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 0.3703 | Regression_cos_phi: 0.2175 | Regression_energy: 0.2204 | Regression_eta: 0.1281 | Regression_pt: 0.2259 | Regression_sin_phi: 0.1788 | LR: 9.87e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.8159s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank3 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank3 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1306 | Classification_binary: 0.3672 | Regression_cos_phi: 0.1571 | Regression_energy: 0.2335 | Regression_eta: 0.1482 | Regression_pt: 0.2414 | Regression_sin_phi: 0.2365 | LR: 9.86e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5853s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank3 [2026-10-28 14:32:39,608] INFO : Running validation on rank3 +rank3 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 14:33:34,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:33:34,721] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 14:34:03,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 14:34:05,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 14:34:09,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 14:34:11,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 14:34:21,317] INFO : Time taken to make predictions on device 3 is: 0.35 min +rank3 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:34:24,282] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 14:34:39,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 14:34:40,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 14:34:43,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 14:34:44,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 14:34:54,643] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:34:56,345] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 14:35:12,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 14:35:12,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 14:35:13,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 14:35:14,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 14:35:24,536] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank3 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1065 | Classification_binary: 0.3079 | Regression_cos_phi: 0.1429 | Regression_energy: 0.1905 | Regression_eta: 0.1090 | Regression_pt: 0.1926 | Regression_sin_phi: 0.1728 | LR: 9.85e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8510s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank3 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank3 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1580 | Classification_binary: 0.3379 | Regression_cos_phi: 0.1862 | Regression_energy: 0.2222 | Regression_eta: 0.1155 | Regression_pt: 0.2295 | Regression_sin_phi: 0.1540 | LR: 9.84e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5089s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank3 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank3 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1612 | Classification_binary: 0.3406 | Regression_cos_phi: 0.1827 | Regression_energy: 0.1767 | Regression_eta: 0.1351 | Regression_pt: 0.1838 | Regression_sin_phi: 0.2182 | LR: 9.84e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4613s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank3 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank3 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1435 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1745 | Regression_energy: 0.1725 | Regression_eta: 0.1086 | Regression_pt: 0.1812 | Regression_sin_phi: 0.1740 | LR: 9.83e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.1732s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank3 [2026-10-28 14:59:28,686] INFO : Step 4500/50000 rank3 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1505 | Classification_binary: 0.3691 | Regression_cos_phi: 0.1634 | Regression_energy: 0.1811 | Regression_eta: 0.1396 | Regression_pt: 0.1932 | Regression_sin_phi: 0.1969 | LR: 9.82e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2507s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank3 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank3 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1532 | Classification_binary: 0.4420 | Regression_cos_phi: 0.2041 | Regression_energy: 0.1858 | Regression_eta: 0.1538 | Regression_pt: 0.1957 | Regression_sin_phi: 0.2181 | LR: 9.81e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7029s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank3 [2026-10-28 15:09:18,560] INFO : Step 4700/50000 rank3 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0978 | Classification_binary: 0.2858 | Regression_cos_phi: 0.1550 | Regression_energy: 0.1734 | Regression_eta: 0.1138 | Regression_pt: 0.1824 | Regression_sin_phi: 0.1628 | LR: 9.81e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7052s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank3 [2026-10-28 15:14:30,754] INFO : Step 4800/50000 rank3 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1434 | Classification_binary: 0.3316 | Regression_cos_phi: 0.2155 | Regression_energy: 0.1901 | Regression_eta: 0.1816 | Regression_pt: 0.1966 | Regression_sin_phi: 0.1788 | LR: 9.80e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4407s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank3 [2026-10-28 15:19:27,570] INFO : Step 4900/50000 rank3 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1228 | Classification_binary: 0.3835 | Regression_cos_phi: 0.1606 | Regression_energy: 0.1825 | Regression_eta: 0.1406 | Regression_pt: 0.1904 | Regression_sin_phi: 0.1934 | LR: 9.79e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5365s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank3 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank3 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1545 | Classification_binary: 0.3631 | Regression_cos_phi: 0.1921 | Regression_energy: 0.1801 | Regression_eta: 0.1376 | Regression_pt: 0.1905 | Regression_sin_phi: 0.1738 | LR: 9.78e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4819s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank3 [2026-10-28 15:24:24,982] INFO : Running validation on rank3 +rank3 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 15:25:19,075] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:25:19,574] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 15:25:44,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 15:25:45,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 15:25:47,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 15:25:48,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 15:25:58,057] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:25:59,251] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 15:26:15,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 15:26:16,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 15:26:17,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 15:26:18,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 15:26:28,136] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 15:26:28,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 15:26:45,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 15:26:46,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 15:26:47,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 15:26:47,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 15:26:57,909] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 15:32:01,930] INFO : Step 5100/50000 rank3 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1118 | Classification_binary: 0.2895 | Regression_cos_phi: 0.2038 | Regression_energy: 0.1861 | Regression_eta: 0.1321 | Regression_pt: 0.1900 | Regression_sin_phi: 0.1577 | LR: 9.77e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.6412s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank3 [2026-10-28 15:36:53,511] INFO : Step 5200/50000 rank3 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1483 | Classification_binary: 0.3776 | Regression_cos_phi: 0.1803 | Regression_energy: 0.1761 | Regression_eta: 0.1604 | Regression_pt: 0.1855 | Regression_sin_phi: 0.1933 | LR: 9.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.6374s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank3 [2026-10-28 15:41:36,527] INFO : Step 5300/50000 rank3 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1321 | Classification_binary: 0.3329 | Regression_cos_phi: 0.2024 | Regression_energy: 0.1717 | Regression_eta: 0.1139 | Regression_pt: 0.1791 | Regression_sin_phi: 0.1671 | LR: 9.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9250s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank3 [2026-10-28 15:46:33,446] INFO : Step 5400/50000 rank3 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.2615 | Regression_cos_phi: 0.1952 | Regression_energy: 0.1392 | Regression_eta: 0.0978 | Regression_pt: 0.1421 | Regression_sin_phi: 0.1387 | LR: 9.74e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4714s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank3 [2026-10-28 15:51:26,877] INFO : Step 5500/50000 rank3 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1461 | Classification_binary: 0.2967 | Regression_cos_phi: 0.1728 | Regression_energy: 0.1589 | Regression_eta: 0.0977 | Regression_pt: 0.1645 | Regression_sin_phi: 0.2019 | LR: 9.73e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.5344s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank3 [2026-10-28 15:56:25,711] INFO : Step 5600/50000 rank3 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1314 | Classification_binary: 0.3922 | Regression_cos_phi: 0.2120 | Regression_energy: 0.1770 | Regression_eta: 0.1399 | Regression_pt: 0.1863 | Regression_sin_phi: 0.1974 | LR: 9.72e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.3200s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank3 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank3 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1207 | Classification_binary: 0.2696 | Regression_cos_phi: 0.1230 | Regression_energy: 0.1721 | Regression_eta: 0.0877 | Regression_pt: 0.1776 | Regression_sin_phi: 0.1449 | LR: 9.71e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5510s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank3 [2026-10-28 16:06:09,459] INFO : Step 5800/50000 rank3 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1195 | Classification_binary: 0.2983 | Regression_cos_phi: 0.2069 | Regression_energy: 0.1544 | Regression_eta: 0.1197 | Regression_pt: 0.1655 | Regression_sin_phi: 0.1645 | LR: 9.70e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.2444s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank3 [2026-10-28 16:11:19,911] INFO : Step 5900/50000 rank3 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.3220 | Regression_cos_phi: 0.1795 | Regression_energy: 0.1821 | Regression_eta: 0.1304 | Regression_pt: 0.1896 | Regression_sin_phi: 0.2171 | LR: 9.69e-04 | DataLoad Time: 0.0156s | Model Forward Time: 7.1587s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank3 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank3 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1246 | Classification_binary: 0.2928 | Regression_cos_phi: 0.1425 | Regression_energy: 0.1687 | Regression_eta: 0.0978 | Regression_pt: 0.1766 | Regression_sin_phi: 0.1391 | LR: 9.68e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4957s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank3 [2026-10-28 16:16:41,623] INFO : Running validation on rank3 +rank3 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:17:32,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:17:33,086] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 16:17:59,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:18:00,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:18:01,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:18:02,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:18:12,655] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:18:13,876] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 16:18:29,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:18:30,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:18:31,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:18:32,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:18:42,976] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:18:43,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:18:43,951] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 16:18:59,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:19:01,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:19:03,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:19:04,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:19:14,216] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank3 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1380 | Classification_binary: 0.2723 | Regression_cos_phi: 0.2236 | Regression_energy: 0.1705 | Regression_eta: 0.1166 | Regression_pt: 0.1740 | Regression_sin_phi: 0.1322 | LR: 9.67e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9554s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank3 [2026-10-28 16:29:26,889] INFO : Step 6200/50000 rank3 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1436 | Classification_binary: 0.3320 | Regression_cos_phi: 0.1519 | Regression_energy: 0.1627 | Regression_eta: 0.1709 | Regression_pt: 0.1699 | Regression_sin_phi: 0.2446 | LR: 9.66e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0907s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank3 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank3 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3664 | Regression_cos_phi: 0.1888 | Regression_energy: 0.1586 | Regression_eta: 0.1625 | Regression_pt: 0.1638 | Regression_sin_phi: 0.1394 | LR: 9.65e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.0299s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank3 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank3 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1126 | Classification_binary: 0.2806 | Regression_cos_phi: 0.1559 | Regression_energy: 0.1643 | Regression_eta: 0.1182 | Regression_pt: 0.1723 | Regression_sin_phi: 0.1516 | LR: 9.64e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7769s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank3 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank3 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2815 | Regression_cos_phi: 0.1088 | Regression_energy: 0.1761 | Regression_eta: 0.0940 | Regression_pt: 0.1848 | Regression_sin_phi: 0.1412 | LR: 9.63e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5004s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank3 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank3 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.3488 | Regression_cos_phi: 0.1446 | Regression_energy: 0.1677 | Regression_eta: 0.1185 | Regression_pt: 0.1721 | Regression_sin_phi: 0.1809 | LR: 9.62e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8074s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank3 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank3 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1248 | Classification_binary: 0.3411 | Regression_cos_phi: 0.1541 | Regression_energy: 0.1564 | Regression_eta: 0.1168 | Regression_pt: 0.1617 | Regression_sin_phi: 0.1907 | LR: 9.61e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.3363s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank3 [2026-10-28 16:58:34,482] INFO : Step 6800/50000 rank3 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.2897 | Regression_cos_phi: 0.1596 | Regression_energy: 0.1335 | Regression_eta: 0.1066 | Regression_pt: 0.1397 | Regression_sin_phi: 0.1463 | LR: 9.60e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.4630s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank3 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank3 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1455 | Classification_binary: 0.2944 | Regression_cos_phi: 0.1728 | Regression_energy: 0.1705 | Regression_eta: 0.1196 | Regression_pt: 0.1760 | Regression_sin_phi: 0.1477 | LR: 9.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6888s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank3 [2026-10-28 17:08:29,914] INFO : Step 7000/50000 rank3 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1419 | Classification_binary: 0.3468 | Regression_cos_phi: 0.1943 | Regression_energy: 0.1539 | Regression_eta: 0.1446 | Regression_pt: 0.1566 | Regression_sin_phi: 0.2104 | LR: 9.57e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9218s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank3 [2026-10-28 17:08:30,791] INFO : Running validation on rank3 +rank3 [2026-10-28 17:08:30,791] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 17:08:30,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 17:09:23,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:09:24,268] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 17:09:51,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 17:09:53,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 17:09:55,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 17:09:56,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 17:10:06,822] INFO : Time taken to make predictions on device 3 is: 0.29 min +rank3 [2026-10-28 17:10:07,735] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:10:08,263] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 17:10:24,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 17:10:25,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 17:10:26,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 17:10:27,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 17:10:37,013] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:10:37,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:10:38,447] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 17:10:54,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 17:10:55,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 17:10:56,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 17:10:57,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 17:11:07,513] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 17:16:27,595] INFO : Step 7100/50000 rank3 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1182 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1844 | Regression_energy: 0.1580 | Regression_eta: 0.1328 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1645 | LR: 9.56e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1476s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank3 [2026-10-28 17:21:34,076] INFO : Step 7200/50000 rank3 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1011 | Classification_binary: 0.2992 | Regression_cos_phi: 0.2093 | Regression_energy: 0.1444 | Regression_eta: 0.1234 | Regression_pt: 0.1504 | Regression_sin_phi: 0.1689 | LR: 9.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3455s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank3 [2026-10-28 17:26:34,943] INFO : Step 7300/50000 rank3 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1236 | Classification_binary: 0.3557 | Regression_cos_phi: 0.2053 | Regression_energy: 0.1494 | Regression_eta: 0.1560 | Regression_pt: 0.1563 | Regression_sin_phi: 0.1710 | LR: 9.53e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4026s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank3 [2026-10-28 17:31:34,684] INFO : Step 7400/50000 rank3 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0992 | Classification_binary: 0.2716 | Regression_cos_phi: 0.1930 | Regression_energy: 0.1557 | Regression_eta: 0.1138 | Regression_pt: 0.1606 | Regression_sin_phi: 0.1914 | LR: 9.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank3 [2026-10-28 17:36:30,885] INFO : Step 7500/50000 rank3 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.2662 | Regression_cos_phi: 0.1667 | Regression_energy: 0.1520 | Regression_eta: 0.1036 | Regression_pt: 0.1625 | Regression_sin_phi: 0.1712 | LR: 9.51e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6606s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank3 [2026-10-28 17:41:09,113] INFO : Step 7600/50000 rank3 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1271 | Classification_binary: 0.3502 | Regression_cos_phi: 0.1451 | Regression_energy: 0.1371 | Regression_eta: 0.1297 | Regression_pt: 0.1428 | Regression_sin_phi: 0.1698 | LR: 9.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7907s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank3 [2026-10-28 17:46:03,244] INFO : Step 7700/50000 rank3 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.3080 | Regression_cos_phi: 0.1247 | Regression_energy: 0.1509 | Regression_eta: 0.1108 | Regression_pt: 0.1570 | Regression_sin_phi: 0.1392 | LR: 9.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7830s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank3 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank3 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.2441 | Regression_cos_phi: 0.1467 | Regression_energy: 0.1405 | Regression_eta: 0.1031 | Regression_pt: 0.1433 | Regression_sin_phi: 0.1740 | LR: 9.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4017s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank3 [2026-10-28 17:56:05,209] INFO : Step 7900/50000 rank3 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2954 | Regression_cos_phi: 0.1424 | Regression_energy: 0.1365 | Regression_eta: 0.0891 | Regression_pt: 0.1418 | Regression_sin_phi: 0.1200 | LR: 9.46e-04 | DataLoad Time: 0.0208s | Model Forward Time: 6.3966s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank3 [2026-10-28 18:00:55,277] INFO : Step 8000/50000 rank3 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.3054 | Regression_cos_phi: 0.1455 | Regression_energy: 0.1336 | Regression_eta: 0.1193 | Regression_pt: 0.1367 | Regression_sin_phi: 0.1714 | LR: 9.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4827s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank3 [2026-10-28 18:00:56,227] INFO : Running validation on rank3 +rank3 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:01:47,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:01:48,315] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 18:02:14,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:02:15,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:02:16,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:02:17,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:02:27,786] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:02:28,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 18:02:44,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:02:45,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:02:46,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:02:47,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:02:57,328] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:02:58,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:02:58,747] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 18:03:14,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:03:15,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:03:16,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:03:17,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:03:27,063] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 18:08:30,021] INFO : Step 8100/50000 rank3 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1188 | Classification_binary: 0.2522 | Regression_cos_phi: 0.1442 | Regression_energy: 0.1340 | Regression_eta: 0.0973 | Regression_pt: 0.1355 | Regression_sin_phi: 0.1237 | LR: 9.43e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.3408s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank3 [2026-10-28 18:13:41,921] INFO : Step 8200/50000 rank3 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1426 | Regression_energy: 0.1373 | Regression_eta: 0.0762 | Regression_pt: 0.1425 | Regression_sin_phi: 0.1294 | LR: 9.42e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3245s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank3 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank3 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1100 | Classification_binary: 0.3689 | Regression_cos_phi: 0.1952 | Regression_energy: 0.1598 | Regression_eta: 0.1230 | Regression_pt: 0.1654 | Regression_sin_phi: 0.1638 | LR: 9.40e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.6366s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank3 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank3 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1141 | Classification_binary: 0.2850 | Regression_cos_phi: 0.1481 | Regression_energy: 0.1414 | Regression_eta: 0.1007 | Regression_pt: 0.1448 | Regression_sin_phi: 0.1488 | LR: 9.39e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1837s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank3 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank3 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1145 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1035 | Regression_energy: 0.1418 | Regression_eta: 0.0872 | Regression_pt: 0.1482 | Regression_sin_phi: 0.1338 | LR: 9.37e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5047s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank3 [2026-10-28 18:34:09,472] INFO : Step 8600/50000 rank3 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1060 | Classification_binary: 0.2595 | Regression_cos_phi: 0.1165 | Regression_energy: 0.1256 | Regression_eta: 0.0876 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0996 | LR: 9.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7735s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank3 [2026-10-28 18:39:14,579] INFO : Step 8700/50000 rank3 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0934 | Classification_binary: 0.3088 | Regression_cos_phi: 0.1386 | Regression_energy: 0.1321 | Regression_eta: 0.1036 | Regression_pt: 0.1365 | Regression_sin_phi: 0.1649 | LR: 9.34e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.7194s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank3 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank3 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1993 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1263 | Regression_eta: 0.0891 | Regression_pt: 0.1304 | Regression_sin_phi: 0.1238 | LR: 9.33e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.4095s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank3 [2026-10-28 18:49:28,516] INFO : Step 8900/50000 rank3 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2833 | Regression_cos_phi: 0.1430 | Regression_energy: 0.1513 | Regression_eta: 0.0811 | Regression_pt: 0.1591 | Regression_sin_phi: 0.1412 | LR: 9.31e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7914s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank3 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank3 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.3385 | Regression_cos_phi: 0.1194 | Regression_energy: 0.1468 | Regression_eta: 0.1033 | Regression_pt: 0.1522 | Regression_sin_phi: 0.1049 | LR: 9.30e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8398s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank3 [2026-10-28 18:54:54,935] INFO : Running validation on rank3 +rank3 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:55:52,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:55:53,257] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 18:56:19,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:56:21,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:56:22,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:56:24,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:56:34,265] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:56:35,473] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 18:56:51,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:56:52,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:56:53,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:56:54,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:57:04,303] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:57:05,961] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 18:57:21,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:57:22,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:57:23,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:57:24,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:57:34,455] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 19:03:08,598] INFO : Step 9100/50000 rank3 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2614 | Regression_cos_phi: 0.1189 | Regression_energy: 0.1221 | Regression_eta: 0.0882 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1433 | LR: 9.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1709s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank3 [2026-10-28 19:08:18,234] INFO : Step 9200/50000 rank3 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0939 | Classification_binary: 0.2722 | Regression_cos_phi: 0.1641 | Regression_energy: 0.1315 | Regression_eta: 0.1076 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1416 | LR: 9.27e-04 | DataLoad Time: 0.0258s | Model Forward Time: 6.5436s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank3 [2026-10-28 19:13:30,253] INFO : Step 9300/50000 rank3 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1128 | Classification_binary: 0.3600 | Regression_cos_phi: 0.1809 | Regression_energy: 0.1496 | Regression_eta: 0.1117 | Regression_pt: 0.1583 | Regression_sin_phi: 0.1525 | LR: 9.25e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3900s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank3 [2026-10-28 19:18:55,614] INFO : Step 9400/50000 rank3 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1165 | Classification_binary: 0.2676 | Regression_cos_phi: 0.1112 | Regression_energy: 0.1272 | Regression_eta: 0.1000 | Regression_pt: 0.1306 | Regression_sin_phi: 0.1651 | LR: 9.24e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2020s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank3 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank3 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2285 | Regression_cos_phi: 0.1195 | Regression_energy: 0.1429 | Regression_eta: 0.0795 | Regression_pt: 0.1475 | Regression_sin_phi: 0.1141 | LR: 9.22e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2838s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank3 [2026-10-28 19:29:34,439] INFO : Step 9600/50000 rank3 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.2566 | Regression_cos_phi: 0.1280 | Regression_energy: 0.1475 | Regression_eta: 0.0894 | Regression_pt: 0.1531 | Regression_sin_phi: 0.1071 | LR: 9.21e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.6056s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank3 [2026-10-28 19:34:53,791] INFO : Step 9700/50000 rank3 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.3174 | Regression_cos_phi: 0.1485 | Regression_energy: 0.1050 | Regression_eta: 0.0972 | Regression_pt: 0.1093 | Regression_sin_phi: 0.1062 | LR: 9.19e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.9770s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank3 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank3 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2213 | Regression_cos_phi: 0.1092 | Regression_energy: 0.1096 | Regression_eta: 0.0821 | Regression_pt: 0.1137 | Regression_sin_phi: 0.1080 | LR: 9.17e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5610s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank3 [2026-10-28 19:44:46,547] INFO : Step 9900/50000 rank3 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0855 | Classification_binary: 0.2606 | Regression_cos_phi: 0.1759 | Regression_energy: 0.1167 | Regression_eta: 0.1103 | Regression_pt: 0.1190 | Regression_sin_phi: 0.1526 | LR: 9.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9313s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank3 [2026-10-28 19:49:37,630] INFO : Step 10000/50000 rank3 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0890 | Classification_binary: 0.3273 | Regression_cos_phi: 0.1734 | Regression_energy: 0.1390 | Regression_eta: 0.1135 | Regression_pt: 0.1437 | Regression_sin_phi: 0.1578 | LR: 9.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6235s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank3 [2026-10-28 19:49:38,690] INFO : Running validation on rank3 +rank3 [2026-10-28 19:49:38,690] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 19:49:38,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 19:50:35,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:50:35,625] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 19:51:01,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 19:51:02,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 19:51:03,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 19:51:04,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 19:51:14,935] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 19:51:15,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:51:16,414] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 19:51:32,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 19:51:33,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 19:51:34,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 19:51:35,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 19:51:45,580] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 19:51:46,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:51:46,686] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 19:52:02,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 19:52:03,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 19:52:04,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 19:52:05,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 19:52:15,208] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 19:57:27,967] INFO : Step 10100/50000 rank3 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2552 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1263 | Regression_eta: 0.0752 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0990 | LR: 9.12e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7455s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank3 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank3 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0860 | Classification_binary: 0.2441 | Regression_cos_phi: 0.1106 | Regression_energy: 0.1169 | Regression_eta: 0.0860 | Regression_pt: 0.1195 | Regression_sin_phi: 0.1106 | LR: 9.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6816s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank3 [2026-10-28 20:07:31,754] INFO : Step 10300/50000 rank3 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0862 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1189 | Regression_eta: 0.1008 | Regression_pt: 0.1236 | Regression_sin_phi: 0.1611 | LR: 9.09e-04 | DataLoad Time: 0.0240s | Model Forward Time: 5.1990s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank3 [2026-10-28 20:12:49,148] INFO : Step 10400/50000 rank3 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2886 | Regression_cos_phi: 0.1307 | Regression_energy: 0.1059 | Regression_eta: 0.0848 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1207 | LR: 9.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2216s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank3 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank3 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1946 | Regression_cos_phi: 0.1187 | Regression_energy: 0.1224 | Regression_eta: 0.0738 | Regression_pt: 0.1290 | Regression_sin_phi: 0.0980 | LR: 9.06e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3223s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank3 [2026-10-28 20:23:17,055] INFO : Step 10600/50000 rank3 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0988 | Classification_binary: 0.2434 | Regression_cos_phi: 0.1105 | Regression_energy: 0.0967 | Regression_eta: 0.0859 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1362 | LR: 9.04e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0928s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank3 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank3 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1383 | Regression_energy: 0.1396 | Regression_eta: 0.0714 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0990 | LR: 9.02e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.8719s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank3 [2026-10-28 20:33:28,996] INFO : Step 10800/50000 rank3 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2055 | Regression_cos_phi: 0.1234 | Regression_energy: 0.1102 | Regression_eta: 0.0860 | Regression_pt: 0.1140 | Regression_sin_phi: 0.1167 | LR: 9.00e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.6819s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank3 [2026-10-28 20:38:44,061] INFO : Step 10900/50000 rank3 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2772 | Regression_cos_phi: 0.1250 | Regression_energy: 0.1001 | Regression_eta: 0.0818 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1261 | LR: 8.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7061s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank3 [2026-10-28 20:43:46,528] INFO : Step 11000/50000 rank3 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2906 | Regression_cos_phi: 0.1213 | Regression_energy: 0.1125 | Regression_eta: 0.0759 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0961 | LR: 8.97e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4456s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank3 [2026-10-28 20:43:47,415] INFO : Running validation on rank3 +rank3 [2026-10-28 20:43:47,415] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 20:44:41,297] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 20:45:09,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 20:45:10,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 20:45:11,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 20:45:12,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 20:45:22,993] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 20:45:23,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:45:24,127] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 20:45:40,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 20:45:41,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 20:45:42,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 20:45:45,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 20:45:55,027] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 20:46:11,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 20:46:13,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 20:46:14,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 20:46:15,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 20:46:25,725] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 20:51:55,950] INFO : Step 11100/50000 rank3 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2273 | Regression_cos_phi: 0.0878 | Regression_energy: 0.1069 | Regression_eta: 0.0670 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0945 | LR: 8.95e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8326s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank3 [2026-10-28 20:57:10,090] INFO : Step 11200/50000 rank3 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0921 | Regression_eta: 0.0516 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0762 | LR: 8.93e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8533s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank3 [2026-10-28 21:02:18,460] INFO : Step 11300/50000 rank3 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 0.2374 | Regression_cos_phi: 0.0923 | Regression_energy: 0.1211 | Regression_eta: 0.0777 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0929 | LR: 8.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2917s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank3 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank3 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2245 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1111 | Regression_eta: 0.0727 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0974 | LR: 8.89e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9850s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank3 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank3 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.2195 | Regression_cos_phi: 0.1677 | Regression_energy: 0.1117 | Regression_eta: 0.0650 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0887 | LR: 8.88e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3391s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank3 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank3 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.2828 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1133 | Regression_eta: 0.0825 | Regression_pt: 0.1165 | Regression_sin_phi: 0.1277 | LR: 8.86e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2582s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank3 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank3 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2592 | Regression_cos_phi: 0.1107 | Regression_energy: 0.1104 | Regression_eta: 0.0677 | Regression_pt: 0.1151 | Regression_sin_phi: 0.1100 | LR: 8.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5932s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank3 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank3 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.2454 | Regression_cos_phi: 0.1143 | Regression_energy: 0.0951 | Regression_eta: 0.0821 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1158 | LR: 8.82e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5600s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank3 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank3 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2207 | Regression_cos_phi: 0.1407 | Regression_energy: 0.1120 | Regression_eta: 0.0795 | Regression_pt: 0.1189 | Regression_sin_phi: 0.1248 | LR: 8.80e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9861s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank3 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank3 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2743 | Regression_cos_phi: 0.0907 | Regression_energy: 0.1119 | Regression_eta: 0.0628 | Regression_pt: 0.1176 | Regression_sin_phi: 0.1066 | LR: 8.78e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0213s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank3 [2026-10-28 21:36:38,362] INFO : Running validation on rank3 +rank3 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:37:31,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 21:38:00,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:38:01,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:38:02,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:38:03,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:38:13,534] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:38:14,611] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 21:38:30,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:38:32,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:38:33,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:38:34,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:38:44,922] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 21:38:45,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:38:46,116] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 21:39:01,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:39:02,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:39:03,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:39:04,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:39:14,238] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank3 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.2380 | Regression_cos_phi: 0.0887 | Regression_energy: 0.1079 | Regression_eta: 0.0630 | Regression_pt: 0.1128 | Regression_sin_phi: 0.1201 | LR: 8.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1510s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank3 [2026-10-28 21:49:00,968] INFO : Step 12200/50000 rank3 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0945 | Regression_eta: 0.0531 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0506 | LR: 8.74e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5158s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank3 [2026-10-28 21:53:55,878] INFO : Step 12300/50000 rank3 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2486 | Regression_cos_phi: 0.1117 | Regression_energy: 0.0935 | Regression_eta: 0.0741 | Regression_pt: 0.0989 | Regression_sin_phi: 0.1128 | LR: 8.72e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6862s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank3 [2026-10-28 21:58:40,948] INFO : Step 12400/50000 rank3 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.3053 | Regression_cos_phi: 0.1199 | Regression_energy: 0.0911 | Regression_eta: 0.0699 | Regression_pt: 0.0952 | Regression_sin_phi: 0.1438 | LR: 8.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2708s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank3 [2026-10-28 22:03:23,385] INFO : Step 12500/50000 rank3 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0950 | Regression_eta: 0.0689 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0788 | LR: 8.68e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6720s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank3 [2026-10-28 22:08:04,804] INFO : Step 12600/50000 rank3 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2235 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0886 | Regression_eta: 0.0625 | Regression_pt: 0.0912 | Regression_sin_phi: 0.1017 | LR: 8.66e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0970s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank3 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank3 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0715 | Regression_eta: 0.0728 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0837 | LR: 8.64e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3606s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank3 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank3 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0671 | Regression_energy: 0.1024 | Regression_eta: 0.0654 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0822 | LR: 8.62e-04 | DataLoad Time: 0.0239s | Model Forward Time: 6.0952s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank3 [2026-10-28 22:22:32,704] INFO : Step 12900/50000 rank3 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0916 | Regression_eta: 0.0575 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0858 | LR: 8.60e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6637s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank3 [2026-10-28 22:27:21,815] INFO : Step 13000/50000 rank3 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2375 | Regression_cos_phi: 0.0964 | Regression_energy: 0.0853 | Regression_eta: 0.0575 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1019 | LR: 8.58e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9248s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank3 [2026-10-28 22:27:23,204] INFO : Running validation on rank3 +rank3 [2026-10-28 22:27:23,204] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 22:27:23,204] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:28:28,170] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 22:28:54,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 22:28:55,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 22:28:56,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 22:28:57,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 22:29:07,226] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:29:08,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:29:08,604] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 22:29:24,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 22:29:25,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 22:29:26,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 22:29:27,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 22:29:37,089] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:29:38,493] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 22:29:54,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 22:29:55,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 22:29:56,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 22:29:56,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 22:30:06,940] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 22:35:15,708] INFO : Step 13100/50000 rank3 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2510 | Regression_cos_phi: 0.1104 | Regression_energy: 0.1017 | Regression_eta: 0.0694 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0988 | LR: 8.56e-04 | DataLoad Time: 0.0222s | Model Forward Time: 6.4839s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank3 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank3 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0843 | Regression_eta: 0.0706 | Regression_pt: 0.0889 | Regression_sin_phi: 0.1084 | LR: 8.54e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6880s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank3 [2026-10-28 22:44:58,539] INFO : Step 13300/50000 rank3 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2505 | Regression_cos_phi: 0.1471 | Regression_energy: 0.0926 | Regression_eta: 0.0730 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0962 | LR: 8.52e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank3 [2026-10-28 22:49:50,191] INFO : Step 13400/50000 rank3 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0943 | Classification_binary: 0.3101 | Regression_cos_phi: 0.1792 | Regression_energy: 0.1393 | Regression_eta: 0.1112 | Regression_pt: 0.1420 | Regression_sin_phi: 0.1618 | LR: 8.50e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.6013s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank3 [2026-10-28 22:54:39,510] INFO : Step 13500/50000 rank3 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2274 | Regression_cos_phi: 0.1151 | Regression_energy: 0.0830 | Regression_eta: 0.0582 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0762 | LR: 8.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4633s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank3 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank3 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2552 | Regression_cos_phi: 0.1044 | Regression_energy: 0.0840 | Regression_eta: 0.0653 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0989 | LR: 8.45e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8491s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank3 [2026-10-28 23:04:05,987] INFO : Step 13700/50000 rank3 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.2853 | Regression_cos_phi: 0.1194 | Regression_energy: 0.1019 | Regression_eta: 0.0678 | Regression_pt: 0.1059 | Regression_sin_phi: 0.1137 | LR: 8.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7306s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank3 [2026-10-28 23:09:16,294] INFO : Step 13800/50000 rank3 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0824 | Regression_eta: 0.0522 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0742 | LR: 8.41e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4802s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank3 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank3 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1020 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0824 | Regression_energy: 0.1030 | Regression_eta: 0.0555 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0860 | LR: 8.39e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2900s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank3 [2026-10-28 23:18:59,723] INFO : Step 14000/50000 rank3 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2401 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0952 | Regression_eta: 0.0608 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0794 | LR: 8.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1765s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank3 [2026-10-28 23:19:00,860] INFO : Running validation on rank3 +rank3 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:19:53,517] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-28 23:20:19,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 23:20:20,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 23:20:21,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 23:20:22,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 23:20:32,615] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 23:20:33,202] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:20:33,744] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-28 23:20:49,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 23:20:50,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 23:20:51,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 23:20:52,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 23:21:02,534] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 23:21:03,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:21:04,038] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-28 23:21:19,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 23:21:20,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 23:21:21,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 23:21:21,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 23:21:32,004] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 23:26:43,452] INFO : Step 14100/50000 rank3 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1166 | Regression_energy: 0.0860 | Regression_eta: 0.0852 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0825 | LR: 8.35e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.5312s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank3 [2026-10-28 23:31:31,721] INFO : Step 14200/50000 rank3 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0866 | Regression_energy: 0.1060 | Regression_eta: 0.0482 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0796 | LR: 8.32e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.6162s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank3 [2026-10-28 23:36:16,487] INFO : Step 14300/50000 rank3 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2266 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0785 | Regression_eta: 0.0745 | Regression_pt: 0.0832 | Regression_sin_phi: 0.1243 | LR: 8.30e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6728s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank3 [2026-10-28 23:40:58,338] INFO : Step 14400/50000 rank3 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2457 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0691 | Regression_eta: 0.0571 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0649 | LR: 8.28e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7776s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank3 [2026-10-28 23:45:42,940] INFO : Step 14500/50000 rank3 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0816 | Regression_eta: 0.0451 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0542 | LR: 8.26e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.6351s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank3 [2026-10-28 23:50:27,493] INFO : Step 14600/50000 rank3 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0863 | Regression_eta: 0.0519 | Regression_pt: 0.0882 | Regression_sin_phi: 0.1030 | LR: 8.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4489s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank3 [2026-10-28 23:55:12,844] INFO : Step 14700/50000 rank3 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2749 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0821 | Regression_eta: 0.0677 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0795 | LR: 8.21e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6458s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank3 [2026-10-28 23:59:56,485] INFO : Step 14800/50000 rank3 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0893 | Regression_eta: 0.0530 | Regression_pt: 0.0932 | Regression_sin_phi: 0.1059 | LR: 8.19e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.7591s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank3 [2026-10-29 00:04:46,437] INFO : Step 14900/50000 rank3 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0744 | Regression_eta: 0.0457 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0761 | LR: 8.17e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank3 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank3 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2557 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0822 | Regression_eta: 0.0597 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0871 | LR: 8.15e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7490s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank3 [2026-10-29 00:09:44,170] INFO : Running validation on rank3 +rank3 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:10:38,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:10:39,314] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 00:11:05,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:11:07,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:11:08,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:11:09,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:11:19,925] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 00:11:20,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:11:20,774] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 00:11:36,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:11:37,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:11:38,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:11:39,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:11:49,564] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:11:50,988] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 00:12:06,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:12:07,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:12:08,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:12:09,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:12:19,419] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 00:17:30,255] INFO : Step 15100/50000 rank3 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2336 | Regression_cos_phi: 0.1132 | Regression_energy: 0.0851 | Regression_eta: 0.0753 | Regression_pt: 0.0899 | Regression_sin_phi: 0.1140 | LR: 8.12e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.3995s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank3 [2026-10-29 00:22:16,918] INFO : Step 15200/50000 rank3 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0829 | Regression_eta: 0.0610 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0805 | LR: 8.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2250s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank3 [2026-10-29 00:27:05,718] INFO : Step 15300/50000 rank3 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2030 | Regression_cos_phi: 0.1084 | Regression_energy: 0.0809 | Regression_eta: 0.0499 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0771 | LR: 8.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3158s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank3 [2026-10-29 00:31:57,727] INFO : Step 15400/50000 rank3 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2517 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0836 | Regression_eta: 0.0572 | Regression_pt: 0.0866 | Regression_sin_phi: 0.1010 | LR: 8.05e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6658s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank3 [2026-10-29 00:36:52,790] INFO : Step 15500/50000 rank3 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0799 | Regression_eta: 0.0522 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0559 | LR: 8.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4216s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank3 [2026-10-29 00:41:49,268] INFO : Step 15600/50000 rank3 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1925 | Regression_cos_phi: 0.1087 | Regression_energy: 0.0784 | Regression_eta: 0.0564 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0827 | LR: 8.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9291s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank3 [2026-10-29 00:46:43,872] INFO : Step 15700/50000 rank3 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2538 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0774 | Regression_eta: 0.0510 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0984 | LR: 7.98e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.7548s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank3 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank3 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2207 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0842 | Regression_eta: 0.0583 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0716 | LR: 7.96e-04 | DataLoad Time: 0.0235s | Model Forward Time: 5.0243s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank3 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank3 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0943 | Regression_energy: 0.0816 | Regression_eta: 0.0662 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0737 | LR: 7.94e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1816s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank3 [2026-10-29 01:01:16,825] INFO : Step 16000/50000 rank3 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2095 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0613 | Regression_eta: 0.0428 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0745 | LR: 7.91e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.3015s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank3 [2026-10-29 01:01:17,882] INFO : Running validation on rank3 +rank3 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 01:02:10,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:02:11,733] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 01:02:39,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 01:02:40,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 01:02:42,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 01:02:44,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 01:02:54,296] INFO : Time taken to make predictions on device 3 is: 0.29 min +rank3 [2026-10-29 01:02:55,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:02:55,636] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 01:03:11,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 01:03:12,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 01:03:13,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 01:03:14,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 01:03:24,530] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 01:03:25,538] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:03:26,089] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 01:03:41,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 01:03:42,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 01:03:43,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 01:03:44,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 01:03:54,330] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank3 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2497 | Regression_cos_phi: 0.1142 | Regression_energy: 0.0901 | Regression_eta: 0.1112 | Regression_pt: 0.0904 | Regression_sin_phi: 0.1016 | LR: 7.89e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0665s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank3 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank3 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2017 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0779 | Regression_eta: 0.0457 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0604 | LR: 7.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2114s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank3 [2026-10-29 01:19:03,892] INFO : Step 16300/50000 rank3 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.2121 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0745 | Regression_eta: 0.0626 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0846 | LR: 7.84e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.2096s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank3 [2026-10-29 01:23:56,184] INFO : Step 16400/50000 rank3 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2837 | Regression_cos_phi: 0.1055 | Regression_energy: 0.0778 | Regression_eta: 0.0776 | Regression_pt: 0.0831 | Regression_sin_phi: 0.1057 | LR: 7.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6840s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank3 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank3 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0824 | Regression_energy: 0.1156 | Regression_eta: 0.0511 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0785 | LR: 7.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3072s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank3 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank3 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0709 | Regression_eta: 0.0555 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0686 | LR: 7.77e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1708s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank3 [2026-10-29 01:38:38,619] INFO : Step 16700/50000 rank3 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2618 | Regression_cos_phi: 0.1101 | Regression_energy: 0.0955 | Regression_eta: 0.0842 | Regression_pt: 0.0980 | Regression_sin_phi: 0.1410 | LR: 7.74e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.9118s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank3 [2026-10-29 01:43:30,909] INFO : Step 16800/50000 rank3 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2171 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1013 | Regression_eta: 0.0665 | Regression_pt: 0.1022 | Regression_sin_phi: 0.1159 | LR: 7.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.5997s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank3 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank3 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0806 | Regression_eta: 0.0357 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0474 | LR: 7.69e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3823s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank3 [2026-10-29 01:53:19,530] INFO : Step 17000/50000 rank3 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0719 | Regression_eta: 0.0469 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0594 | LR: 7.67e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.6662s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank3 [2026-10-29 01:53:20,625] INFO : Running validation on rank3 +rank3 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 01:54:26,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:54:27,391] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 01:55:04,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 01:55:05,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 01:55:06,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 01:55:07,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 01:55:17,906] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:55:19,053] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 01:55:34,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 01:55:35,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 01:55:37,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 01:55:38,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 01:55:48,664] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:55:50,414] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 01:56:05,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 01:56:06,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 01:56:07,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 01:56:08,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 01:56:18,377] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 02:01:32,243] INFO : Step 17100/50000 rank3 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.2592 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0718 | Regression_eta: 0.0593 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0627 | LR: 7.64e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.1159s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank3 [2026-10-29 02:06:24,514] INFO : Step 17200/50000 rank3 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1938 | Regression_cos_phi: 0.1156 | Regression_energy: 0.0611 | Regression_eta: 0.0435 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0869 | LR: 7.62e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.6089s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank3 [2026-10-29 02:11:06,514] INFO : Step 17300/50000 rank3 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0629 | Regression_eta: 0.0511 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0652 | LR: 7.59e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7534s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank3 [2026-10-29 02:16:07,514] INFO : Step 17400/50000 rank3 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.2423 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0692 | Regression_eta: 0.0713 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0884 | LR: 7.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5461s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank3 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank3 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2092 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0894 | Regression_eta: 0.0490 | Regression_pt: 0.0903 | Regression_sin_phi: 0.1077 | LR: 7.54e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.2413s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank3 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank3 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1776 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0703 | Regression_eta: 0.0606 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0812 | LR: 7.52e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4711s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank3 [2026-10-29 02:30:47,052] INFO : Step 17700/50000 rank3 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0840 | Regression_energy: 0.0806 | Regression_eta: 0.0613 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0775 | LR: 7.49e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.7715s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank3 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank3 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0877 | Regression_eta: 0.0632 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0901 | LR: 7.47e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0210s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank3 [2026-10-29 02:40:30,747] INFO : Step 17900/50000 rank3 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0552 | Regression_eta: 0.0526 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0757 | LR: 7.44e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5934s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank3 [2026-10-29 02:45:16,090] INFO : Step 18000/50000 rank3 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0633 | Regression_eta: 0.0573 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0671 | LR: 7.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6576s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank3 [2026-10-29 02:45:17,190] INFO : Running validation on rank3 +rank3 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:46:12,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:46:13,322] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 02:46:39,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:46:40,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:46:41,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:46:42,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:46:52,573] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:46:53,610] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 02:47:09,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:47:10,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:47:12,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:47:14,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:47:24,599] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 02:47:25,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:47:25,821] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 02:47:42,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:47:43,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:47:43,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:47:44,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:47:54,729] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank3 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2400 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0684 | Regression_eta: 0.0630 | Regression_pt: 0.0708 | Regression_sin_phi: 0.1136 | LR: 7.39e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7395s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank3 [2026-10-29 02:58:08,644] INFO : Step 18200/50000 rank3 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0906 | Regression_energy: 0.0915 | Regression_eta: 0.0526 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0555 | LR: 7.36e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.9651s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank3 [2026-10-29 03:02:55,066] INFO : Step 18300/50000 rank3 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0816 | Regression_eta: 0.0427 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0756 | LR: 7.34e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.5931s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank3 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank3 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0694 | Regression_eta: 0.0470 | Regression_pt: 0.0739 | Regression_sin_phi: 0.1107 | LR: 7.31e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7120s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank3 [2026-10-29 03:12:43,565] INFO : Step 18500/50000 rank3 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0746 | Regression_eta: 0.0473 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0719 | LR: 7.29e-04 | DataLoad Time: 0.0128s | Model Forward Time: 5.3954s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank3 [2026-10-29 03:17:39,475] INFO : Step 18600/50000 rank3 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0762 | Regression_eta: 0.0457 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0618 | LR: 7.26e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4737s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank3 [2026-10-29 03:22:31,157] INFO : Step 18700/50000 rank3 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0802 | Regression_eta: 0.0418 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0569 | LR: 7.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4794s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank3 [2026-10-29 03:27:37,245] INFO : Step 18800/50000 rank3 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0702 | Regression_eta: 0.0562 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0989 | LR: 7.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.9528s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank3 [2026-10-29 03:32:26,943] INFO : Step 18900/50000 rank3 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0631 | Regression_eta: 0.0453 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0748 | LR: 7.18e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5372s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank3 [2026-10-29 03:37:06,568] INFO : Step 19000/50000 rank3 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0783 | Regression_eta: 0.0488 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0549 | LR: 7.16e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5324s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank3 [2026-10-29 03:37:07,713] INFO : Running validation on rank3 +rank3 [2026-10-29 03:37:07,713] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 03:37:07,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 03:38:00,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:38:00,821] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 03:38:26,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 03:38:28,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 03:38:29,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 03:38:30,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 03:38:40,306] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 03:38:40,770] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:38:41,425] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 03:38:57,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 03:38:58,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 03:38:59,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 03:39:00,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 03:39:10,238] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:39:11,226] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 03:39:26,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 03:39:27,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 03:39:28,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 03:39:29,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 03:39:39,310] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank3 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0701 | Regression_eta: 0.0703 | Regression_pt: 0.0727 | Regression_sin_phi: 0.1058 | LR: 7.13e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank3 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank3 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0645 | Regression_eta: 0.0403 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0774 | LR: 7.10e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5604s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank3 [2026-10-29 03:54:09,159] INFO : Step 19300/50000 rank3 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0692 | Regression_eta: 0.0633 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0793 | LR: 7.08e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3962s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank3 [2026-10-29 03:58:51,216] INFO : Step 19400/50000 rank3 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0603 | Regression_eta: 0.0529 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0982 | LR: 7.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6158s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank3 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank3 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0745 | Regression_eta: 0.0542 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0867 | LR: 7.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank3 [2026-10-29 04:08:12,467] INFO : Step 19600/50000 rank3 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0848 | Regression_eta: 0.0373 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0439 | LR: 7.00e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.4205s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank3 [2026-10-29 04:12:52,348] INFO : Step 19700/50000 rank3 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0568 | Regression_eta: 0.0454 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0785 | LR: 6.97e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3574s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank3 [2026-10-29 04:17:28,314] INFO : Step 19800/50000 rank3 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0664 | Regression_eta: 0.0504 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0912 | LR: 6.94e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.0473s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank3 [2026-10-29 04:22:08,871] INFO : Step 19900/50000 rank3 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0802 | Regression_eta: 0.0645 | Regression_pt: 0.0848 | Regression_sin_phi: 0.1054 | LR: 6.92e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.3175s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank3 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank3 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0709 | Regression_eta: 0.0701 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0746 | LR: 6.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6547s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank3 [2026-10-29 04:26:52,970] INFO : Running validation on rank3 +rank3 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:27:46,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:27:46,651] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 04:28:14,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:28:15,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:28:16,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:28:17,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:28:27,301] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:28:28,500] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 04:28:44,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:28:45,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:28:46,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:28:47,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:28:57,643] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:28:59,247] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 04:29:14,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:29:15,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:29:16,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:29:17,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:29:27,533] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 04:34:27,055] INFO : Step 20100/50000 rank3 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2082 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0792 | Regression_eta: 0.0696 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0668 | LR: 6.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.3251s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank3 [2026-10-29 04:39:08,147] INFO : Step 20200/50000 rank3 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0716 | Regression_eta: 0.0368 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0752 | LR: 6.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6170s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank3 [2026-10-29 04:43:47,102] INFO : Step 20300/50000 rank3 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0620 | Regression_eta: 0.0398 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0699 | LR: 6.81e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4949s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank3 [2026-10-29 04:48:26,732] INFO : Step 20400/50000 rank3 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0721 | Regression_eta: 0.0584 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0676 | LR: 6.78e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4434s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank3 [2026-10-29 04:53:08,466] INFO : Step 20500/50000 rank3 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0906 | Regression_energy: 0.0581 | Regression_eta: 0.0858 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0642 | LR: 6.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0623s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank3 [2026-10-29 04:57:53,290] INFO : Step 20600/50000 rank3 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0658 | Regression_eta: 0.0411 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0520 | LR: 6.73e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7516s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank3 [2026-10-29 05:02:47,889] INFO : Step 20700/50000 rank3 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0653 | Regression_eta: 0.0440 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0802 | LR: 6.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6373s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank3 [2026-10-29 05:07:34,619] INFO : Step 20800/50000 rank3 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0740 | Regression_eta: 0.0487 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0583 | LR: 6.67e-04 | DataLoad Time: 0.0192s | Model Forward Time: 6.8149s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank3 [2026-10-29 05:12:18,487] INFO : Step 20900/50000 rank3 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0593 | Regression_eta: 0.0376 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0575 | LR: 6.65e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6295s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank3 [2026-10-29 05:17:03,998] INFO : Step 21000/50000 rank3 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0706 | Regression_eta: 0.0563 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0877 | LR: 6.62e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5932s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank3 [2026-10-29 05:17:04,883] INFO : Running validation on rank3 +rank3 [2026-10-29 05:17:04,883] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 05:17:58,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:17:58,802] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 05:18:23,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 05:18:24,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 05:18:25,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 05:18:26,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 05:18:36,106] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:18:37,118] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 05:18:53,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 05:18:54,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 05:18:55,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 05:18:56,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 05:19:06,082] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 05:19:06,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:19:07,296] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 05:19:22,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 05:19:23,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 05:19:24,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 05:19:25,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 05:19:35,426] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank3 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0653 | Regression_eta: 0.0537 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0532 | LR: 6.59e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.3795s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank3 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank3 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 05:29:24,523] INFO : Step 21200/50000 rank3 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0562 | Regression_eta: 0.0407 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0742 | LR: 6.56e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.6427s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank3 [2026-10-29 05:34:39,115] INFO : Step 21300/50000 rank3 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0597 | Regression_eta: 0.0501 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0642 | LR: 6.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6723s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank3 [2026-10-29 05:39:29,689] INFO : Step 21400/50000 rank3 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2329 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0620 | Regression_eta: 0.0493 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0835 | LR: 6.51e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.3968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank3 [2026-10-29 05:44:17,735] INFO : Step 21500/50000 rank3 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0632 | Regression_eta: 0.0330 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0646 | LR: 6.48e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3087s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank3 [2026-10-29 05:49:11,032] INFO : Step 21600/50000 rank3 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0725 | Regression_eta: 0.0370 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0786 | LR: 6.45e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5084s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank3 [2026-10-29 05:53:59,566] INFO : Step 21700/50000 rank3 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2129 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0753 | Regression_eta: 0.0584 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0830 | LR: 6.43e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.5565s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank3 [2026-10-29 05:58:52,460] INFO : Step 21800/50000 rank3 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0612 | Regression_eta: 0.0471 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0874 | LR: 6.40e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.1505s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank3 [2026-10-29 06:03:49,408] INFO : Step 21900/50000 rank3 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0719 | Regression_eta: 0.0599 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0680 | LR: 6.37e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8517s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank3 [2026-10-29 06:08:50,709] INFO : Step 22000/50000 rank3 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2019 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0683 | Regression_eta: 0.0496 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0771 | LR: 6.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4725s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank3 [2026-10-29 06:08:51,802] INFO : Running validation on rank3 +rank3 [2026-10-29 06:08:51,802] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 06:08:51,802] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 06:09:45,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:09:46,124] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 06:10:11,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 06:10:12,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 06:10:14,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 06:10:15,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 06:10:25,314] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 06:10:26,929] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 06:10:43,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 06:10:44,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 06:10:45,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 06:10:45,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 06:10:55,989] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:10:56,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:10:57,442] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 06:11:13,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 06:11:13,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 06:11:14,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 06:11:15,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 06:11:25,611] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank3 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0598 | Regression_eta: 0.0510 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0741 | LR: 6.32e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1897s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank3 [2026-10-29 06:21:31,723] INFO : Step 22200/50000 rank3 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1454 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0678 | Regression_eta: 0.0657 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0944 | LR: 6.29e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.2934s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank3 [2026-10-29 06:26:33,414] INFO : Step 22300/50000 rank3 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0579 | Regression_eta: 0.0518 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0568 | LR: 6.26e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7523s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank3 [2026-10-29 06:31:29,808] INFO : Step 22400/50000 rank3 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.2120 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0542 | Regression_eta: 0.0456 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0749 | LR: 6.23e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.4017s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank3 [2026-10-29 06:36:31,514] INFO : Step 22500/50000 rank3 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0767 | Regression_eta: 0.0471 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0696 | LR: 6.20e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.2542s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank3 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank3 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0541 | Regression_eta: 0.0516 | Regression_pt: 0.0562 | Regression_sin_phi: 0.1024 | LR: 6.18e-04 | DataLoad Time: 0.0256s | Model Forward Time: 4.3114s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank3 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank3 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0835 | Regression_eta: 0.0774 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0738 | LR: 6.15e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.9421s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank3 [2026-10-29 06:51:26,820] INFO : Step 22800/50000 rank3 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0651 | Regression_eta: 0.0415 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0816 | LR: 6.12e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.4047s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank3 [2026-10-29 06:56:23,353] INFO : Step 22900/50000 rank3 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0712 | Regression_eta: 0.0539 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0614 | LR: 6.09e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8112s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank3 [2026-10-29 07:01:33,064] INFO : Step 23000/50000 rank3 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.2260 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0579 | Regression_eta: 0.0587 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0836 | LR: 6.06e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4762s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank3 [2026-10-29 07:01:34,091] INFO : Running validation on rank3 +rank3 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:02:28,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 07:02:49,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:02:50,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:02:51,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:02:52,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:03:02,736] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:03:03,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:03:03,679] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 07:03:19,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:03:20,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:03:21,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:03:22,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:03:32,730] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:03:33,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:03:34,171] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 07:03:49,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:03:50,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:03:51,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:03:52,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:04:02,504] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 07:09:13,970] INFO : Step 23100/50000 rank3 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0636 | Regression_eta: 0.0513 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0629 | LR: 6.04e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1785s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank3 [2026-10-29 07:14:12,718] INFO : Step 23200/50000 rank3 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0592 | Regression_eta: 0.0257 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0537 | LR: 6.01e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6122s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank3 [2026-10-29 07:19:05,797] INFO : Step 23300/50000 rank3 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0762 | Regression_eta: 0.0469 | Regression_pt: 0.0804 | Regression_sin_phi: 0.1510 | LR: 5.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8581s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank3 [2026-10-29 07:24:01,131] INFO : Step 23400/50000 rank3 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0687 | Regression_eta: 0.0568 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0877 | LR: 5.95e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0100s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank3 [2026-10-29 07:29:14,711] INFO : Step 23500/50000 rank3 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0619 | Regression_eta: 0.0475 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0616 | LR: 5.92e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8333s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank3 [2026-10-29 07:34:22,319] INFO : Step 23600/50000 rank3 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0951 | Regression_eta: 0.0703 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0656 | LR: 5.90e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.3908s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank3 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank3 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0558 | Regression_eta: 0.0481 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0857 | LR: 5.87e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.8165s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank3 [2026-10-29 07:44:46,683] INFO : Step 23800/50000 rank3 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0716 | Regression_eta: 0.0583 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0760 | LR: 5.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8777s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank3 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank3 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0276 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0590 | Regression_eta: 0.0360 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0593 | LR: 5.81e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.8154s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank3 [2026-10-29 07:54:58,899] INFO : Step 24000/50000 rank3 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0653 | Regression_eta: 0.0439 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0804 | LR: 5.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8557s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank3 [2026-10-29 07:54:59,832] INFO : Running validation on rank3 +rank3 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:55:54,113] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 07:56:20,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:56:22,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:56:23,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:56:24,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:56:34,113] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:56:35,052] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 07:56:51,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:56:52,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:56:52,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:56:53,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:57:03,943] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:57:05,695] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 07:57:21,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:57:22,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:57:23,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:57:24,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:57:34,169] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 08:03:04,663] INFO : Step 24100/50000 rank3 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2181 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0637 | Regression_eta: 0.0544 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0697 | LR: 5.75e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5906s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank3 [2026-10-29 08:07:59,793] INFO : Step 24200/50000 rank3 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0624 | Regression_eta: 0.0509 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0784 | LR: 5.73e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0935s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank3 [2026-10-29 08:13:18,515] INFO : Step 24300/50000 rank3 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0698 | Regression_energy: 0.1102 | Regression_eta: 0.0451 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0746 | LR: 5.70e-04 | DataLoad Time: 0.0200s | Model Forward Time: 7.4989s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank3 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank3 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0690 | Regression_eta: 0.0500 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0536 | LR: 5.67e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1592s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank3 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank3 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0606 | Regression_eta: 0.0397 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0530 | LR: 5.64e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5210s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank3 [2026-10-29 08:28:18,514] INFO : Step 24600/50000 rank3 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0652 | Regression_eta: 0.0885 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0588 | LR: 5.61e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7047s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank3 [2026-10-29 08:33:21,873] INFO : Step 24700/50000 rank3 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0599 | Regression_eta: 0.0607 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0599 | LR: 5.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9278s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank3 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank3 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0589 | Regression_eta: 0.0556 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0727 | LR: 5.56e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2077s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank3 [2026-10-29 08:43:21,524] INFO : Step 24900/50000 rank3 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0538 | Regression_eta: 0.0310 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0587 | LR: 5.53e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4735s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank3 [2026-10-29 08:48:25,834] INFO : Step 25000/50000 rank3 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0542 | Regression_eta: 0.0393 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0627 | LR: 5.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9518s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank3 [2026-10-29 08:48:26,771] INFO : Running validation on rank3 +rank3 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 08:49:20,639] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:49:21,192] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 08:49:51,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 08:49:53,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 08:49:54,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 08:49:55,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 08:50:05,556] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:50:06,729] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 08:50:22,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 08:50:23,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 08:50:24,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 08:50:25,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 08:50:35,771] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:50:36,988] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 08:50:52,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 08:50:53,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 08:50:54,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 08:50:55,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 08:51:05,408] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 08:56:18,538] INFO : Step 25100/50000 rank3 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0639 | Regression_eta: 0.0479 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0560 | LR: 5.47e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.3788s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank3 [2026-10-29 09:01:01,591] INFO : Step 25200/50000 rank3 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0588 | Regression_eta: 0.0540 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0539 | LR: 5.44e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8754s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank3 [2026-10-29 09:05:50,664] INFO : Step 25300/50000 rank3 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0745 | Regression_eta: 0.0368 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0670 | LR: 5.42e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9780s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank3 [2026-10-29 09:10:30,565] INFO : Step 25400/50000 rank3 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0581 | Regression_eta: 0.0429 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0616 | LR: 5.39e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.1707s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank3 [2026-10-29 09:15:11,238] INFO : Step 25500/50000 rank3 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0533 | Regression_eta: 0.0475 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0497 | LR: 5.36e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4699s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank3 [2026-10-29 09:19:56,606] INFO : Step 25600/50000 rank3 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0538 | Regression_eta: 0.0497 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0691 | LR: 5.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.9368s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank3 [2026-10-29 09:24:33,865] INFO : Step 25700/50000 rank3 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0628 | Regression_eta: 0.0435 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0574 | LR: 5.30e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2128s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank3 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank3 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1989 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0692 | Regression_eta: 0.0412 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0713 | LR: 5.27e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1659s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank3 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank3 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0516 | Regression_eta: 0.0338 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0453 | LR: 5.25e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.7716s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank3 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank3 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0637 | Regression_eta: 0.0705 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0658 | LR: 5.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7492s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank3 [2026-10-29 09:39:04,382] INFO : Running validation on rank3 +rank3 [2026-10-29 09:39:04,382] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:39:04,382] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:39:55,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:39:56,249] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 09:40:23,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:40:24,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:40:25,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:40:26,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:40:36,454] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:40:37,507] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 09:40:54,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:40:56,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:40:57,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:40:58,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:41:08,214] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 09:41:08,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:44:09,582] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 09:44:25,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:44:26,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:44:27,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:44:28,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:44:38,018] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 09:50:29,555] INFO : Step 26100/50000 rank3 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0606 | Regression_eta: 0.0550 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0655 | LR: 5.19e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.1779s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank3 [2026-10-29 09:55:15,274] INFO : Step 26200/50000 rank3 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0521 | Regression_eta: 0.0319 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0480 | LR: 5.16e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8144s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank3 [2026-10-29 09:59:57,745] INFO : Step 26300/50000 rank3 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1122 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0637 | Regression_eta: 0.0222 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0662 | LR: 5.13e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.9318s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank3 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank3 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0548 | Regression_eta: 0.0526 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0727 | LR: 5.10e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.1487s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank3 [2026-10-29 10:09:23,739] INFO : Step 26500/50000 rank3 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0639 | Regression_eta: 0.0495 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0682 | LR: 5.08e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0942s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank3 [2026-10-29 10:14:05,782] INFO : Step 26600/50000 rank3 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1260 | Regression_cos_phi: 0.0351 | Regression_energy: 0.0735 | Regression_eta: 0.0359 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0480 | LR: 5.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6626s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank3 [2026-10-29 10:19:02,169] INFO : Step 26700/50000 rank3 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0643 | Regression_eta: 0.0380 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0527 | LR: 5.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2161s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank3 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank3 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2217 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0745 | Regression_eta: 0.0565 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0600 | LR: 4.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5300s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank3 [2026-10-29 10:28:51,387] INFO : Step 26900/50000 rank3 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0198 | Classification_binary: 0.1089 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0559 | Regression_eta: 0.0256 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0324 | LR: 4.96e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.5626s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank3 [2026-10-29 10:33:37,219] INFO : Step 27000/50000 rank3 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0844 | Regression_eta: 0.0369 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0696 | LR: 4.94e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.7029s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank3 [2026-10-29 10:33:38,214] INFO : Running validation on rank3 +rank3 [2026-10-29 10:33:38,214] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 10:33:38,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 10:34:33,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:34:34,225] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 10:35:00,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 10:35:01,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 10:35:02,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 10:35:03,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 10:35:13,323] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:35:13,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:35:14,237] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 10:35:30,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 10:35:31,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 10:35:32,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 10:35:33,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 10:35:43,349] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:35:44,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:35:45,039] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 10:36:01,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 10:36:01,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 10:36:02,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 10:36:03,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 10:36:13,678] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 10:41:28,285] INFO : Step 27100/50000 rank3 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0481 | Regression_eta: 0.0399 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0638 | LR: 4.91e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6740s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank3 [2026-10-29 10:46:22,294] INFO : Step 27200/50000 rank3 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0501 | Regression_energy: 0.1050 | Regression_eta: 0.0637 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0370 | LR: 4.88e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8256s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank3 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank3 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0615 | Regression_eta: 0.0334 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0439 | LR: 4.85e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6695s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank3 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank3 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0570 | Regression_eta: 0.0362 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0583 | LR: 4.82e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9762s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank3 [2026-10-29 11:01:35,253] INFO : Step 27500/50000 rank3 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0568 | Regression_eta: 0.0333 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0502 | LR: 4.80e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.3137s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank3 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank3 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0500 | Regression_eta: 0.0308 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0615 | LR: 4.77e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.3325s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank3 [2026-10-29 11:13:33,057] INFO : Step 27700/50000 rank3 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0612 | Regression_eta: 0.0302 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0588 | LR: 4.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7947s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank3 [2026-10-29 11:18:30,826] INFO : Step 27800/50000 rank3 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.2096 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0560 | Regression_eta: 0.0516 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0597 | LR: 4.71e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0129s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank3 [2026-10-29 11:23:38,362] INFO : Step 27900/50000 rank3 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0643 | Regression_eta: 0.0512 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0811 | LR: 4.68e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8865s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank3 [2026-10-29 11:28:54,872] INFO : Step 28000/50000 rank3 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0597 | Regression_eta: 0.0417 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0751 | LR: 4.66e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.9091s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank3 [2026-10-29 11:28:55,940] INFO : Running validation on rank3 +rank3 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 11:29:47,976] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:29:48,520] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 11:30:14,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 11:30:15,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 11:30:16,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 11:30:17,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 11:30:27,735] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:30:28,727] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 11:30:44,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 11:30:45,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 11:30:46,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 11:30:47,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 11:30:57,746] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 11:30:57,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 11:31:14,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 11:31:15,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 11:31:16,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 11:31:17,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 11:31:27,042] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 11:37:01,729] INFO : Step 28100/50000 rank3 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0653 | Regression_eta: 0.0413 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0465 | LR: 4.63e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.5015s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank3 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank3 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0599 | Regression_eta: 0.0357 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0481 | LR: 4.60e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.0746s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank3 [2026-10-29 11:47:09,275] INFO : Step 28300/50000 rank3 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0303 | Classification_binary: 0.1061 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0597 | Regression_eta: 0.0428 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0461 | LR: 4.57e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0920s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank3 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank3 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1910 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1345 | Regression_eta: 0.0709 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0862 | LR: 4.55e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank3 [2026-10-29 11:56:56,842] INFO : Step 28500/50000 rank3 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0599 | Regression_eta: 0.0321 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0611 | LR: 4.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9905s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank3 [2026-10-29 12:01:46,916] INFO : Step 28600/50000 rank3 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0600 | Regression_eta: 0.0422 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0711 | LR: 4.49e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6580s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank3 [2026-10-29 12:06:39,994] INFO : Step 28700/50000 rank3 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0667 | Regression_eta: 0.0512 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0767 | LR: 4.46e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5138s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank3 [2026-10-29 12:11:31,024] INFO : Step 28800/50000 rank3 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0610 | Regression_eta: 0.0386 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0525 | LR: 4.44e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.9328s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank3 [2026-10-29 12:16:17,097] INFO : Step 28900/50000 rank3 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0617 | Regression_eta: 0.0343 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0364 | LR: 4.41e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7003s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank3 [2026-10-29 12:21:10,134] INFO : Step 29000/50000 rank3 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0699 | Regression_eta: 0.0444 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0587 | LR: 4.38e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9921s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank3 [2026-10-29 12:21:11,218] INFO : Running validation on rank3 +rank3 [2026-10-29 12:21:11,218] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 12:21:11,218] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 12:22:31,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 12:22:32,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 12:22:33,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 12:22:34,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 12:22:44,881] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:22:45,975] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 12:23:01,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 12:23:02,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 12:23:03,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 12:23:04,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 12:23:14,881] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 12:23:15,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:23:16,443] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 12:23:32,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 12:23:33,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 12:23:34,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 12:23:35,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 12:23:45,118] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank3 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1928 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0652 | Regression_eta: 0.0464 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0620 | LR: 4.35e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1721s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank3 [2026-10-29 12:33:43,194] INFO : Step 29200/50000 rank3 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0632 | Regression_eta: 0.0324 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0451 | LR: 4.33e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5663s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank3 [2026-10-29 14:49:26,825] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-29 14:49:26,831] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank3 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-29 14:49:54,089] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-29 14:49:54,090] INFO : nn0 parameters: 171776 +rank3 [2026-10-29 14:49:54,090] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-29 14:49:54,090] INFO : Initializing backbone layer 0 +rank3 [2026-10-29 14:49:54,127] INFO : Initializing backbone layer 1 +rank3 [2026-10-29 14:49:54,141] INFO : Initializing backbone layer 2 +rank3 [2026-10-29 14:49:54,156] INFO : Initializing backbone layer 3 +rank3 [2026-10-29 14:49:54,170] INFO : Initializing backbone layer 4 +rank3 [2026-10-29 14:49:54,184] INFO : Initializing backbone layer 5 +rank3 [2026-10-29 14:49:54,194] INFO : Backbone initialization took 0.10s +rank3 [2026-10-29 14:49:54,194] INFO : backbone parameters: 7901376 +rank3 [2026-10-29 14:49:54,194] INFO : Initializing output DNNs +rank3 [2026-10-29 14:49:54,228] INFO : Output DNNs initialization took 0.03s +rank3 [2026-10-29 14:49:54,228] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-29 14:49:54,228] INFO : jagged_attention enabled=False +rank3 [2026-10-29 14:49:54,228] INFO : flash_attn_varlen enabled=False +rank3 [2026-10-29 14:49:54,228] INFO : task_queries enabled=False +rank3 [2026-10-29 14:49:54,228] INFO : classification_norm parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : regression_norm parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : classification_query parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : regression_query parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : classification_readout parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : regression_readout parameters: 0 +rank3 [2026-10-29 14:49:54,228] INFO : output_mode=set +rank3 [2026-10-29 14:49:54,229] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-29 14:49:54,229] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_pid parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_pt parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_eta parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-29 14:49:54,229] INFO : nn_energy parameters: 0 +rank3 [2026-10-29 14:49:54,230] INFO : Total MLPF parameters: 12786964 +rank3 [2026-10-29 14:49:54,230] INFO : MLPF __init__ done +rank3 [2026-10-29 14:49:54,230] INFO : Instantiated model +rank3 [2026-10-29 14:49:54,230] INFO : Moving model to device rank=3 +rank3 [2026-10-29 14:49:54,250] INFO : Moved model to device rank=3 +rank3 [2026-10-29 14:49:54,252] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-29 14:49:54,791] INFO : Restoring training from step 29001 +rank3 [2026-10-29 14:49:54,799] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank3 [2026-10-29 14:49:54,800] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-29 14:49:55,512] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank3 [2026-10-29 14:49:55,513] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-29 14:49:55,514] INFO : Trainable parameters: 12786964 +rank3 [2026-10-29 14:49:55,514] INFO : Non-trainable parameters: 23268 +rank3 [2026-10-29 14:49:55,514] INFO : Total parameters: 12810232 +rank3 [2026-10-29 14:49:55,518] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-29 14:49:55,518] INFO : Getting dataloaders +rank3 [2026-10-29 14:49:55,518] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:01,364] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:02,634] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:03,549] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-29 14:50:03,549] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-29 14:50:03,549] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:03,819] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:04,089] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:50:04,343] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-29 14:50:04,343] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-29 14:50:18,085] INFO : Got dataloaders +rank3 [2026-10-29 14:50:18,085] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank3 [2026-10-29 14:50:18,085] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank3 [2026-10-29 14:50:18,085] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank3 [2026-10-29 14:50:18,085] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank3 [2026-10-29 14:50:19,623] INFO : loader split=train rank=3 len=21108 +rank3 [2026-10-29 14:50:19,623] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-29 14:50:19,623] INFO : Starting training from step 29001 to 50000 on rank 3 of 8 +rank3 [2026-10-29 14:50:19,624] INFO : Creating train_iterator +rank3 [2026-10-29 14:50:19,624] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank3 [2026-10-29 14:50:33,778] INFO : Created train_iterator +rank3 [2026-10-29 14:55:12,919] INFO : Step 29100/50000 rank3 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0610 | Regression_eta: 0.0496 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0684 | LR: 4.35e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7160s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank3 [2026-10-29 14:59:46,788] INFO : Step 29200/50000 rank3 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0532 | Regression_eta: 0.0308 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0369 | LR: 4.33e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3625s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank3 [2026-10-29 15:04:21,833] INFO : Step 29300/50000 rank3 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1198 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0660 | Regression_eta: 0.0403 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0649 | LR: 4.30e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0917s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank3 [2026-10-29 15:08:57,031] INFO : Step 29400/50000 rank3 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0588 | Regression_eta: 0.0351 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0635 | LR: 4.27e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2732s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank3 [2026-10-29 15:13:31,011] INFO : Step 29500/50000 rank3 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0564 | Regression_eta: 0.0379 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0566 | LR: 4.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4985s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank3 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank3 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0686 | Regression_eta: 0.0388 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0575 | LR: 4.22e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5919s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank3 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank3 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1100 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0528 | Regression_eta: 0.0298 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0394 | LR: 4.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6729s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank3 [2026-10-29 15:27:18,562] INFO : Step 29800/50000 rank3 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0443 | Regression_eta: 0.0488 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0838 | LR: 4.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6304s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank3 [2026-10-29 15:31:51,911] INFO : Step 29900/50000 rank3 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0531 | Regression_eta: 0.0326 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0564 | LR: 4.14e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5834s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank3 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank3 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0493 | Regression_eta: 0.0346 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0432 | LR: 4.11e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4522s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank3 [2026-10-29 15:36:27,739] INFO : Running validation on rank3 +rank3 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 15:37:19,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:37:19,772] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 15:37:50,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 15:37:51,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 15:37:52,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 15:37:53,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 15:38:03,943] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:38:04,945] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 15:38:20,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 15:38:21,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 15:38:22,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 15:38:23,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 15:38:33,397] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:38:35,048] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 15:38:50,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 15:38:51,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 15:38:52,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 15:38:52,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 15:39:02,925] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 15:44:19,186] INFO : Step 30100/50000 rank3 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0636 | Regression_eta: 0.0451 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0706 | LR: 4.08e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.7126s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank3 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank3 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0682 | Regression_eta: 0.0346 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0614 | LR: 4.06e-04 | DataLoad Time: 0.0265s | Model Forward Time: 4.7631s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank3 [2026-10-29 15:53:38,867] INFO : Step 30300/50000 rank3 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0537 | Regression_eta: 0.0446 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0473 | LR: 4.03e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3376s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank3 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank3 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0688 | Regression_eta: 0.0493 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0611 | LR: 4.00e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.4500s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank3 [2026-10-29 16:02:51,486] INFO : Step 30500/50000 rank3 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2323 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0643 | Regression_eta: 0.0368 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0582 | LR: 3.98e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.4583s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank3 [2026-10-29 16:07:32,049] INFO : Step 30600/50000 rank3 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0505 | Regression_eta: 0.0365 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0429 | LR: 3.95e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.9361s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank3 [2026-10-29 16:12:18,780] INFO : Step 30700/50000 rank3 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1334 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0578 | Regression_eta: 0.0401 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0728 | LR: 3.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2830s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank3 [2026-10-29 16:17:03,635] INFO : Step 30800/50000 rank3 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0526 | Regression_eta: 0.0454 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0623 | LR: 3.90e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0639s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank3 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank3 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0654 | Regression_eta: 0.0333 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0651 | LR: 3.87e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6943s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank3 [2026-10-29 16:26:02,769] INFO : Step 31000/50000 rank3 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1059 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0550 | Regression_eta: 0.0363 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0425 | LR: 3.84e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0260s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank3 [2026-10-29 16:26:49,213] INFO : Running validation on rank3 +rank3 [2026-10-29 16:26:49,213] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:26:49,213] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:27:39,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:27:40,304] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 16:28:07,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:28:09,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:28:10,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:28:12,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:28:22,608] INFO : Time taken to make predictions on device 3 is: 0.29 min +rank3 [2026-10-29 16:28:24,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:28:24,725] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 16:28:40,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:28:41,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:28:42,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:28:43,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:28:53,238] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:28:54,191] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:28:54,677] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 16:29:10,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:29:11,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:29:12,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:29:13,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:29:23,108] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank3 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0441 | Regression_eta: 0.0366 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0661 | LR: 3.82e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.5517s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank3 [2026-10-29 16:39:13,624] INFO : Step 31200/50000 rank3 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0678 | Regression_eta: 0.0592 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0675 | LR: 3.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0727s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank3 [2026-10-29 16:43:57,174] INFO : Step 31300/50000 rank3 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0496 | Regression_eta: 0.0228 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0576 | LR: 3.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8847s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank3 [2026-10-29 16:48:30,666] INFO : Step 31400/50000 rank3 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0574 | Regression_eta: 0.0370 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0446 | LR: 3.74e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7836s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank3 [2026-10-29 16:53:07,751] INFO : Step 31500/50000 rank3 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0487 | Regression_eta: 0.0441 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0449 | LR: 3.71e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank3 [2026-10-29 16:57:38,607] INFO : Step 31600/50000 rank3 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0699 | Regression_eta: 0.0344 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0779 | LR: 3.69e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1666s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank3 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank3 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0432 | Regression_eta: 0.0380 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0510 | LR: 3.66e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1002s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank3 [2026-10-29 17:06:47,524] INFO : Step 31800/50000 rank3 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0501 | Regression_eta: 0.0349 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0589 | LR: 3.64e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2355s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank3 [2026-10-29 17:11:27,372] INFO : Step 31900/50000 rank3 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0553 | Regression_eta: 0.0332 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0617 | LR: 3.61e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5206s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank3 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank3 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0534 | Regression_eta: 0.0282 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0428 | LR: 3.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4949s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank3 [2026-10-29 17:16:12,935] INFO : Running validation on rank3 +rank3 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 17:17:03,853] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:17:04,359] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 17:17:32,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 17:17:33,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 17:17:34,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 17:17:35,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 17:17:45,135] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:17:46,318] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 17:18:02,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 17:18:03,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 17:18:04,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 17:18:05,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 17:18:15,111] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:18:15,920] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:18:16,464] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 17:18:32,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 17:18:33,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 17:18:34,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 17:18:34,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 17:18:44,926] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank3 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0602 | Regression_eta: 0.0423 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0646 | LR: 3.56e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9451s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank3 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank3 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0508 | Regression_eta: 0.0435 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0631 | LR: 3.53e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4541s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank3 [2026-10-29 17:33:21,417] INFO : Step 32300/50000 rank3 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0605 | Regression_eta: 0.0532 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0843 | LR: 3.51e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.9284s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank3 [2026-10-29 17:38:08,350] INFO : Step 32400/50000 rank3 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1170 | Regression_cos_phi: 0.1115 | Regression_energy: 0.0473 | Regression_eta: 0.0334 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0846 | LR: 3.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8160s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank3 [2026-10-29 17:42:52,783] INFO : Step 32500/50000 rank3 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0464 | Regression_eta: 0.0413 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0464 | LR: 3.46e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8588s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank3 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank3 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0501 | Regression_eta: 0.0386 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0588 | LR: 3.43e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6203s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank3 [2026-10-29 17:52:17,189] INFO : Step 32700/50000 rank3 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0700 | Regression_eta: 0.0411 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0709 | LR: 3.41e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank3 [2026-10-29 17:57:01,870] INFO : Step 32800/50000 rank3 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0492 | Regression_eta: 0.0381 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0670 | LR: 3.38e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5891s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank3 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank3 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0457 | Regression_eta: 0.0468 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0484 | LR: 3.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4259s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank3 [2026-10-29 18:06:22,611] INFO : Step 33000/50000 rank3 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0532 | Regression_eta: 0.0341 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0508 | LR: 3.33e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2117s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank3 [2026-10-29 18:06:23,516] INFO : Running validation on rank3 +rank3 [2026-10-29 18:06:23,516] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:06:23,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:07:15,766] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 18:07:43,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:07:44,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:07:45,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:07:46,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:07:56,955] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:07:58,401] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 18:08:14,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:08:15,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:08:16,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:08:17,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:08:27,231] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:08:28,344] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 18:08:43,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:08:44,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:08:45,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:08:46,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:08:56,452] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank3 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0532 | Regression_eta: 0.0461 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0468 | LR: 3.31e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2611s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank3 [2026-10-29 18:18:33,981] INFO : Step 33200/50000 rank3 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0703 | Regression_eta: 0.0336 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0465 | LR: 3.28e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4332s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank3 [2026-10-29 18:23:15,420] INFO : Step 33300/50000 rank3 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0524 | Regression_eta: 0.0371 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0368 | LR: 3.26e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6015s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank3 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank3 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1156 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0461 | Regression_eta: 0.0462 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0501 | LR: 3.23e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.2642s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank3 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank3 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0617 | Regression_eta: 0.0547 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0884 | LR: 3.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3717s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank3 [2026-10-29 18:37:27,752] INFO : Step 33600/50000 rank3 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0665 | Regression_eta: 0.0358 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0852 | LR: 3.19e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3474s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank3 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank3 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1069 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0464 | Regression_eta: 0.0206 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0399 | LR: 3.16e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3184s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank3 [2026-10-29 18:47:04,986] INFO : Step 33800/50000 rank3 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0470 | Regression_eta: 0.0369 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0474 | LR: 3.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3942s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank3 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank3 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0570 | Regression_eta: 0.0402 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0699 | LR: 3.11e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1925s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank3 [2026-10-29 18:56:35,807] INFO : Step 34000/50000 rank3 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0513 | Regression_eta: 0.0355 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0488 | LR: 3.09e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1626s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank3 [2026-10-29 18:56:36,806] INFO : Running validation on rank3 +rank3 [2026-10-29 18:56:36,806] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:56:36,806] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:57:29,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:57:30,355] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 18:57:55,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:57:56,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:57:57,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:57:58,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:58:08,144] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:58:09,211] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 18:58:25,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:58:26,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:58:27,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:58:27,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:58:37,986] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:58:39,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 18:58:55,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:58:56,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:58:57,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:58:58,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:59:08,100] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 19:04:16,497] INFO : Step 34100/50000 rank3 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0545 | Regression_eta: 0.0353 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0477 | LR: 3.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8496s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank3 [2026-10-29 19:09:14,637] INFO : Step 34200/50000 rank3 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0456 | Regression_eta: 0.0474 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0658 | LR: 3.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6994s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank3 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank3 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0516 | Regression_eta: 0.0339 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0607 | LR: 3.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3177s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank3 [2026-10-29 19:18:48,203] INFO : Step 34400/50000 rank3 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0640 | Regression_eta: 0.0300 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0571 | LR: 2.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9239s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank3 [2026-10-29 19:23:49,389] INFO : Step 34500/50000 rank3 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0547 | Regression_eta: 0.0410 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0557 | LR: 2.97e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.2317s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank3 [2026-10-29 19:28:34,774] INFO : Step 34600/50000 rank3 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0503 | Regression_eta: 0.0347 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0512 | LR: 2.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4738s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank3 [2026-10-29 19:33:35,743] INFO : Step 34700/50000 rank3 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0518 | Regression_eta: 0.0337 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0472 | LR: 2.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1496s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank3 [2026-10-29 19:38:26,588] INFO : Step 34800/50000 rank3 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0536 | Regression_eta: 0.0239 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0459 | LR: 2.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0541s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank3 [2026-10-29 19:43:15,573] INFO : Step 34900/50000 rank3 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0480 | Regression_eta: 0.0300 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0558 | LR: 2.88e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2776s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank3 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank3 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0263 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0484 | Regression_eta: 0.0349 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0532 | LR: 2.85e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3855s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank3 [2026-10-29 19:48:21,334] INFO : Running validation on rank3 +rank3 [2026-10-29 19:48:21,334] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 19:49:15,447] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:49:15,988] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 19:49:42,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 19:49:43,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 19:49:44,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 19:49:45,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 19:49:55,624] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:49:56,740] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 19:50:12,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 19:50:13,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 19:50:14,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 19:50:15,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 19:50:25,734] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:50:26,883] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 19:50:42,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 19:50:43,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 19:50:44,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 19:50:45,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 19:50:55,182] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 19:56:07,676] INFO : Step 35100/50000 rank3 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1079 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0456 | Regression_eta: 0.0428 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0709 | LR: 2.83e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4696s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank3 [2026-10-29 20:01:01,618] INFO : Step 35200/50000 rank3 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0450 | Regression_eta: 0.0405 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0554 | LR: 2.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6703s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank3 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank3 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0607 | Regression_eta: 0.0292 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0626 | LR: 2.79e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1443s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank3 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank3 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0492 | Regression_eta: 0.0484 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0919 | LR: 2.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4639s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank3 [2026-10-29 20:15:42,289] INFO : Step 35500/50000 rank3 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0448 | Regression_eta: 0.0348 | Regression_pt: 0.0471 | Regression_sin_phi: 0.1019 | LR: 2.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3583s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank3 [2026-10-29 20:20:38,046] INFO : Step 35600/50000 rank3 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0442 | Regression_eta: 0.0300 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0491 | LR: 2.72e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1917s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank3 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank3 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0519 | Regression_eta: 0.0450 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0761 | LR: 2.70e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1735s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank3 [2026-10-29 20:30:22,090] INFO : Step 35800/50000 rank3 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0521 | Regression_eta: 0.0421 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0757 | LR: 2.68e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2799s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank3 [2026-10-29 20:35:20,189] INFO : Step 35900/50000 rank3 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0474 | Regression_eta: 0.0369 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0717 | LR: 2.65e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6685s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank3 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank3 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0437 | Regression_eta: 0.0244 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0491 | LR: 2.63e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2577s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank3 [2026-10-29 20:40:17,059] INFO : Running validation on rank3 +rank3 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 20:41:08,401] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:41:08,961] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 20:41:36,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 20:41:37,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 20:41:38,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 20:41:39,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 20:41:49,103] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:41:50,341] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 20:42:06,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 20:42:07,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 20:42:08,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 20:42:09,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 20:42:19,373] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:42:20,694] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 20:42:36,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 20:42:37,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 20:42:38,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 20:42:39,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 20:42:49,202] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 20:47:46,990] INFO : Step 36100/50000 rank3 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0608 | Regression_eta: 0.0435 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0636 | LR: 2.61e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4503s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank3 [2026-10-29 20:52:22,841] INFO : Step 36200/50000 rank3 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0511 | Regression_eta: 0.0664 | Regression_pt: 0.0531 | Regression_sin_phi: 0.1146 | LR: 2.59e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3974s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank3 [2026-10-29 20:56:59,585] INFO : Step 36300/50000 rank3 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0446 | Regression_eta: 0.0338 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0365 | LR: 2.57e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2289s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank3 [2026-10-29 21:01:38,352] INFO : Step 36400/50000 rank3 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0629 | Regression_eta: 0.0328 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0382 | LR: 2.55e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8768s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank3 [2026-10-29 21:06:14,246] INFO : Step 36500/50000 rank3 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0481 | Regression_eta: 0.0517 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0681 | LR: 2.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4219s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank3 [2026-10-29 21:10:49,732] INFO : Step 36600/50000 rank3 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0375 | Regression_eta: 0.0400 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0508 | LR: 2.50e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2777s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank3 [2026-10-29 21:15:35,108] INFO : Step 36700/50000 rank3 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1007 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0635 | Regression_eta: 0.0285 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0606 | LR: 2.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2704s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank3 [2026-10-29 21:20:28,003] INFO : Step 36800/50000 rank3 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0561 | Regression_eta: 0.0367 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0386 | LR: 2.46e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3994s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank3 [2026-10-29 21:25:23,681] INFO : Step 36900/50000 rank3 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0315 | Regression_energy: 0.0518 | Regression_eta: 0.0275 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0462 | LR: 2.44e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9351s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank3 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank3 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0573 | Regression_eta: 0.0321 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0343 | LR: 2.42e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.9307s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank3 [2026-10-29 21:30:17,322] INFO : Running validation on rank3 +rank3 [2026-10-29 21:30:17,322] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:31:08,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:31:08,620] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 21:31:35,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:31:36,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:31:37,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:31:38,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:31:48,388] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:31:49,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 21:32:05,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:32:06,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:32:07,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:32:08,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:32:18,845] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:32:20,811] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 21:32:36,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:32:37,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:32:37,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:32:38,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:32:48,781] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:37:56,197] INFO : Step 37100/50000 rank3 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.1152 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0477 | Regression_eta: 0.0322 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0388 | LR: 2.40e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6908s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank3 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank3 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0418 | Regression_eta: 0.0404 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0434 | LR: 2.38e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6960s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank3 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank3 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0504 | Regression_eta: 0.0393 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0621 | LR: 2.36e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5582s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank3 [2026-10-29 21:52:56,911] INFO : Step 37400/50000 rank3 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0316 | Regression_energy: 0.0481 | Regression_eta: 0.0385 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0396 | LR: 2.34e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6118s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank3 [2026-10-29 21:57:43,878] INFO : Step 37500/50000 rank3 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0492 | Regression_eta: 0.0446 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0612 | LR: 2.32e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7106s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank3 [2026-10-29 22:02:29,841] INFO : Step 37600/50000 rank3 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0509 | Regression_eta: 0.0284 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0446 | LR: 2.30e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2486s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank3 [2026-10-29 22:07:05,610] INFO : Step 37700/50000 rank3 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1083 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0490 | Regression_eta: 0.0279 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0348 | LR: 2.28e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1562s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank3 [2026-10-29 22:11:38,775] INFO : Step 37800/50000 rank3 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0615 | Regression_eta: 0.0271 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0490 | LR: 2.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2579s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank3 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank3 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0284 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0511 | Regression_eta: 0.0293 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0628 | LR: 2.24e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1254s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank3 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank3 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0185 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0508 | Regression_eta: 0.0238 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0360 | LR: 2.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2882s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank3 [2026-10-29 22:20:57,876] INFO : Running validation on rank3 +rank3 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 22:21:48,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:21:48,922] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 22:22:15,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 22:22:16,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 22:22:17,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 22:22:18,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 22:22:28,382] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 22:22:28,859] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:22:29,427] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 22:22:45,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 22:22:46,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 22:22:47,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 22:22:48,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 22:22:58,222] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:22:59,394] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 22:23:14,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 22:23:15,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 22:23:16,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 22:23:17,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 22:23:27,312] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank3 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0579 | Regression_eta: 0.0384 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0758 | LR: 2.20e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2317s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank3 [2026-10-29 22:33:07,241] INFO : Step 38200/50000 rank3 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0445 | Regression_eta: 0.0444 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0478 | LR: 2.18e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7610s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank3 [2026-10-29 22:37:42,242] INFO : Step 38300/50000 rank3 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0483 | Regression_eta: 0.0331 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0499 | LR: 2.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank3 [2026-10-29 22:42:17,609] INFO : Step 38400/50000 rank3 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0402 | Regression_eta: 0.0333 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0230 | LR: 2.14e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4887s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank3 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank3 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0431 | Regression_eta: 0.0338 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0420 | LR: 2.12e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5202s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank3 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank3 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1922 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0516 | Regression_eta: 0.0503 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0832 | LR: 2.11e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.7275s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank3 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank3 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0258 | Classification_binary: 0.1137 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0458 | Regression_eta: 0.0238 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0444 | LR: 2.09e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1559s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank3 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank3 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1027 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0453 | Regression_eta: 0.0202 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0405 | LR: 2.07e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3990s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank3 [2026-10-29 23:07:29,433] INFO : Step 38900/50000 rank3 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0542 | Regression_eta: 0.0285 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0453 | LR: 2.05e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.8967s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank3 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank3 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0504 | Regression_eta: 0.0356 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0454 | LR: 2.03e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank3 [2026-10-29 23:12:38,865] INFO : Running validation on rank3 +rank3 [2026-10-29 23:12:38,865] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 23:12:38,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 23:13:33,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:13:34,229] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-29 23:14:06,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 23:14:07,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 23:14:08,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 23:14:09,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 23:14:19,483] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 23:14:20,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:14:20,783] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-29 23:14:36,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 23:14:37,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 23:14:38,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 23:14:39,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 23:14:49,848] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:14:50,898] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-29 23:15:06,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 23:15:07,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 23:15:08,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 23:15:09,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 23:15:19,244] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank3 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1024 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0447 | Regression_eta: 0.0282 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0334 | LR: 2.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9864s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank3 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank3 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1322 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0467 | Regression_eta: 0.0355 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0481 | LR: 2.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5913s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank3 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank3 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0548 | Regression_eta: 0.0408 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0391 | LR: 1.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6305s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank3 [2026-10-29 23:34:45,026] INFO : Step 39400/50000 rank3 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.0973 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0446 | Regression_eta: 0.0258 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0371 | LR: 1.96e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.6687s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank3 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank3 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0298 | Classification_binary: 0.1027 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0490 | Regression_eta: 0.0221 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0568 | LR: 1.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4409s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank3 [2026-10-29 23:43:57,880] INFO : Step 39600/50000 rank3 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0291 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0335 | Regression_energy: 0.0384 | Regression_eta: 0.0216 | Regression_pt: 0.0394 | Regression_sin_phi: 0.0360 | LR: 1.93e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.4701s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank3 [2026-10-29 23:48:42,957] INFO : Step 39700/50000 rank3 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0368 | Regression_eta: 0.0434 | Regression_pt: 0.0370 | Regression_sin_phi: 0.0900 | LR: 1.91e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4058s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank3 [2026-10-29 23:53:54,346] INFO : Step 39800/50000 rank3 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1027 | Regression_cos_phi: 0.0365 | Regression_energy: 0.0486 | Regression_eta: 0.0238 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0343 | LR: 1.89e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5388s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank3 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank3 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0447 | Regression_eta: 0.0370 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0491 | LR: 1.88e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7655s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank3 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank3 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0492 | Regression_eta: 0.0406 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0491 | LR: 1.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9277s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank3 [2026-10-30 00:05:56,735] INFO : Running validation on rank3 +rank3 [2026-10-30 00:05:56,735] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:05:56,735] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:06:50,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:06:51,353] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 00:07:17,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:07:18,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:07:19,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:07:20,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:07:30,850] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:07:31,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 00:07:48,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:07:49,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:07:50,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:07:51,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:08:01,117] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:08:01,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 00:08:17,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 00:08:18,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 00:08:19,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 00:08:20,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 00:08:30,260] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank3 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1100 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0448 | Regression_eta: 0.0292 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0654 | LR: 1.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6417s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank3 [2026-10-30 00:18:43,887] INFO : Step 40200/50000 rank3 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1072 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0360 | Regression_eta: 0.0311 | Regression_pt: 0.0360 | Regression_sin_phi: 0.0436 | LR: 1.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4446s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank3 [2026-10-30 00:23:35,704] INFO : Step 40300/50000 rank3 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0297 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0409 | Regression_eta: 0.0274 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0331 | LR: 1.81e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5347s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank3 [2026-10-30 00:28:38,369] INFO : Step 40400/50000 rank3 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0430 | Regression_eta: 0.0444 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0504 | LR: 1.79e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2233s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank3 [2026-10-30 00:33:25,153] INFO : Step 40500/50000 rank3 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1046 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0488 | Regression_eta: 0.0312 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0691 | LR: 1.78e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2277s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank3 [2026-10-30 00:38:07,832] INFO : Step 40600/50000 rank3 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0439 | Regression_eta: 0.0307 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0432 | LR: 1.76e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2976s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank3 [2026-10-30 00:42:54,584] INFO : Step 40700/50000 rank3 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0330 | Regression_energy: 0.0494 | Regression_eta: 0.0215 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0336 | LR: 1.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank3 [2026-10-30 00:47:52,347] INFO : Step 40800/50000 rank3 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0530 | Regression_eta: 0.0232 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0488 | LR: 1.73e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8891s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank3 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank3 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0359 | Regression_eta: 0.0285 | Regression_pt: 0.0367 | Regression_sin_phi: 0.0585 | LR: 1.72e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7070s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank3 [2026-10-30 00:57:38,929] INFO : Step 41000/50000 rank3 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0474 | Regression_eta: 0.0233 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0352 | LR: 1.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3865s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank3 [2026-10-30 00:57:39,931] INFO : Running validation on rank3 +rank3 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:58:31,732] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:58:32,295] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 00:58:59,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:59:00,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:59:01,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:59:02,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:59:12,424] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:59:13,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:59:13,670] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 00:59:29,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:59:30,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:59:31,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:59:32,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:59:42,292] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:59:43,889] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 00:59:59,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:00:00,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:00:01,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:00:02,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:00:12,332] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:05:10,996] INFO : Step 41100/50000 rank3 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0520 | Regression_eta: 0.0296 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0445 | LR: 1.69e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7655s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank3 [2026-10-30 01:10:09,653] INFO : Step 41200/50000 rank3 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0351 | Regression_energy: 0.0709 | Regression_eta: 0.0215 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0423 | LR: 1.67e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6350s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank3 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank3 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0456 | Regression_eta: 0.0323 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0586 | LR: 1.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2625s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank3 [2026-10-30 01:19:55,521] INFO : Step 41400/50000 rank3 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1025 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0514 | Regression_eta: 0.0188 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0286 | LR: 1.64e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.6733s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank3 [2026-10-30 01:24:55,601] INFO : Step 41500/50000 rank3 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.0942 | Regression_cos_phi: 0.0267 | Regression_energy: 0.0397 | Regression_eta: 0.0255 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0478 | LR: 1.63e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.4497s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank3 [2026-10-30 01:29:39,760] INFO : Step 41600/50000 rank3 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0453 | Regression_eta: 0.0399 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0792 | LR: 1.61e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0150s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank3 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank3 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0508 | Regression_eta: 0.0251 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0379 | LR: 1.60e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7046s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank3 [2026-10-30 01:39:10,940] INFO : Step 41800/50000 rank3 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0404 | Regression_eta: 0.0371 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0470 | LR: 1.58e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8810s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank3 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank3 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0261 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0381 | Regression_eta: 0.0272 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0533 | LR: 1.57e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2476s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank3 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank3 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0536 | Regression_eta: 0.0272 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0417 | LR: 1.56e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2699s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank3 [2026-10-30 01:49:03,869] INFO : Running validation on rank3 +rank3 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 01:49:57,369] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 01:50:24,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 01:50:25,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 01:50:27,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 01:50:28,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 01:50:38,072] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:50:39,487] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 01:50:55,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 01:50:56,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 01:50:57,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 01:50:58,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 01:51:08,105] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:51:10,102] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 01:51:26,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:51:26,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:51:27,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:51:28,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:51:38,593] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:57:02,472] INFO : Step 42100/50000 rank3 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0487 | Regression_eta: 0.0482 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0544 | LR: 1.54e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0040s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank3 [2026-10-30 02:01:47,468] INFO : Step 42200/50000 rank3 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0402 | Regression_eta: 0.0330 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0455 | LR: 1.53e-04 | DataLoad Time: 0.0174s | Model Forward Time: 7.2723s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank3 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank3 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-30 02:06:41,299] INFO : Step 42300/50000 rank3 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0461 | Regression_eta: 0.0275 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0407 | LR: 1.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8870s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank3 [2026-10-30 02:11:46,427] INFO : Step 42400/50000 rank3 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0481 | Regression_eta: 0.0217 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0479 | LR: 1.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.1608s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank3 [2026-10-30 02:16:38,985] INFO : Step 42500/50000 rank3 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0470 | Regression_eta: 0.0296 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0454 | LR: 1.49e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8535s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank3 [2026-10-30 02:21:27,109] INFO : Step 42600/50000 rank3 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0480 | Regression_eta: 0.0279 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0434 | LR: 1.48e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.3548s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank3 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank3 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1019 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0603 | Regression_eta: 0.0215 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0355 | LR: 1.47e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6227s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank3 [2026-10-30 02:31:38,797] INFO : Step 42800/50000 rank3 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.0978 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0343 | Regression_eta: 0.0142 | Regression_pt: 0.0349 | Regression_sin_phi: 0.0355 | LR: 1.45e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6414s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank3 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank3 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0388 | Regression_eta: 0.0331 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0244 | LR: 1.44e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8537s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank3 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank3 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1244 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0458 | Regression_eta: 0.0242 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0445 | LR: 1.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5949s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank3 [2026-10-30 02:41:14,878] INFO : Running validation on rank3 +rank3 [2026-10-30 02:41:14,878] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 02:41:14,878] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 02:42:10,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 02:42:38,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 02:42:39,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 02:42:40,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 02:42:41,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 02:42:51,272] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 02:42:51,953] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:42:52,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 02:43:08,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 02:43:09,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 02:43:10,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 02:43:11,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 02:43:21,849] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 02:43:39,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 02:43:40,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 02:43:40,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 02:43:41,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 02:43:51,729] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 02:48:51,377] INFO : Step 43100/50000 rank3 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0244 | Classification_binary: 0.1169 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0452 | Regression_eta: 0.0287 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0644 | LR: 1.42e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9476s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank3 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank3 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0439 | Regression_eta: 0.0316 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0429 | LR: 1.40e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5629s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank3 [2026-10-30 02:58:07,502] INFO : Step 43300/50000 rank3 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0467 | Regression_eta: 0.0324 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0378 | LR: 1.39e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0494s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank3 [2026-10-30 03:02:42,808] INFO : Step 43400/50000 rank3 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0238 | Classification_binary: 0.1135 | Regression_cos_phi: 0.0313 | Regression_energy: 0.0485 | Regression_eta: 0.0371 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0409 | LR: 1.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4741s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank3 [2026-10-30 03:07:16,795] INFO : Step 43500/50000 rank3 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1139 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0439 | Regression_eta: 0.0234 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0382 | LR: 1.37e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9329s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank3 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank3 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0958 | Regression_energy: 0.0387 | Regression_eta: 0.0568 | Regression_pt: 0.0393 | Regression_sin_phi: 0.1111 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8415s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank3 [2026-10-30 03:16:33,743] INFO : Step 43700/50000 rank3 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0469 | Regression_eta: 0.0300 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0346 | LR: 1.35e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1357s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank3 [2026-10-30 03:21:10,514] INFO : Step 43800/50000 rank3 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0522 | Regression_eta: 0.0350 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0674 | LR: 1.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2537s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank3 [2026-10-30 03:26:01,001] INFO : Step 43900/50000 rank3 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0496 | Regression_eta: 0.0377 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0440 | LR: 1.33e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.9706s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank3 [2026-10-30 03:30:36,934] INFO : Step 44000/50000 rank3 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0287 | Regression_energy: 0.0458 | Regression_eta: 0.0262 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0392 | LR: 1.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8191s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank3 [2026-10-30 03:30:37,937] INFO : Running validation on rank3 +rank3 [2026-10-30 03:30:37,937] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 03:30:37,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 03:31:28,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:31:29,164] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 03:31:57,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 03:31:58,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 03:31:59,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 03:32:00,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 03:32:10,787] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 03:32:27,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 03:32:28,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 03:32:29,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 03:32:30,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 03:32:40,443] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:32:41,628] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 03:32:57,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 03:32:58,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 03:32:59,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 03:33:00,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 03:33:10,084] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 03:38:15,293] INFO : Step 44100/50000 rank3 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.0944 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0518 | Regression_eta: 0.0429 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0590 | LR: 1.31e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7479s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank3 [2026-10-30 03:43:04,522] INFO : Step 44200/50000 rank3 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0421 | Regression_eta: 0.0268 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0352 | LR: 1.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4213s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank3 [2026-10-30 03:47:46,465] INFO : Step 44300/50000 rank3 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0425 | Regression_eta: 0.0244 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0594 | LR: 1.29e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4334s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank3 [2026-10-30 03:52:24,793] INFO : Step 44400/50000 rank3 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.0967 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0520 | Regression_eta: 0.0209 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0389 | LR: 1.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3365s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank3 [2026-10-30 03:57:11,971] INFO : Step 44500/50000 rank3 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0282 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0465 | Regression_eta: 0.0280 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0400 | LR: 1.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3159s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank3 [2026-10-30 04:01:48,196] INFO : Step 44600/50000 rank3 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0478 | Regression_eta: 0.0270 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0563 | LR: 1.26e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank3 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank3 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0505 | Regression_eta: 0.0230 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0413 | LR: 1.25e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.6218s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank3 [2026-10-30 04:11:07,220] INFO : Step 44800/50000 rank3 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1042 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0505 | Regression_eta: 0.0260 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0431 | LR: 1.24e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5867s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank3 [2026-10-30 04:15:54,537] INFO : Step 44900/50000 rank3 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0404 | Regression_eta: 0.0273 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0398 | LR: 1.23e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6722s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank3 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank3 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0420 | Regression_eta: 0.0262 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0398 | LR: 1.22e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7897s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank3 [2026-10-30 04:20:37,108] INFO : Running validation on rank3 +rank3 [2026-10-30 04:20:37,108] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 04:20:37,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 04:21:27,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:21:28,558] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 04:21:56,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 04:21:57,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 04:21:58,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 04:21:59,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 04:22:09,704] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:22:10,891] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 04:22:26,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 04:22:27,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 04:22:28,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 04:22:29,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 04:22:39,672] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:22:41,054] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 04:22:56,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 04:22:57,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 04:22:58,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 04:22:59,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 04:23:09,480] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank3 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0249 | Classification_binary: 0.0845 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0452 | Regression_eta: 0.0271 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0464 | LR: 1.21e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2095s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank3 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank3 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0378 | Regression_eta: 0.0311 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0599 | LR: 1.20e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4479s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank3 [2026-10-30 04:37:52,085] INFO : Step 45300/50000 rank3 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0276 | Regression_energy: 0.0456 | Regression_eta: 0.0253 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0364 | LR: 1.19e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8151s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank3 [2026-10-30 04:42:38,426] INFO : Step 45400/50000 rank3 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0424 | Regression_eta: 0.0309 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0440 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4016s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank3 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank3 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0398 | Regression_eta: 0.0429 | Regression_pt: 0.0403 | Regression_sin_phi: 0.0550 | LR: 1.18e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3863s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank3 [2026-10-30 04:52:16,190] INFO : Step 45600/50000 rank3 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0259 | Regression_energy: 0.0423 | Regression_eta: 0.0244 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0289 | LR: 1.17e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4798s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank3 [2026-10-30 04:57:07,845] INFO : Step 45700/50000 rank3 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1140 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0391 | Regression_eta: 0.0234 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0381 | LR: 1.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4897s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank3 [2026-10-30 05:01:56,082] INFO : Step 45800/50000 rank3 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1091 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0367 | Regression_eta: 0.0438 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0695 | LR: 1.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6473s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank3 [2026-10-30 05:06:42,063] INFO : Step 45900/50000 rank3 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0483 | Regression_eta: 0.0378 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0670 | LR: 1.15e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2885s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank3 [2026-10-30 05:11:36,040] INFO : Step 46000/50000 rank3 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0558 | Regression_eta: 0.0229 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0385 | LR: 1.14e-04 | DataLoad Time: 0.0220s | Model Forward Time: 6.4329s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank3 [2026-10-30 05:11:37,037] INFO : Running validation on rank3 +rank3 [2026-10-30 05:11:37,037] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 05:11:37,037] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 05:12:27,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:12:27,993] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 05:12:53,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 05:12:54,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 05:12:55,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 05:12:56,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 05:13:06,168] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 05:13:23,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 05:13:24,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 05:13:24,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 05:13:25,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 05:13:35,947] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 05:13:36,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 05:13:52,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 05:13:53,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 05:13:54,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 05:13:55,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 05:14:05,509] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 05:19:00,537] INFO : Step 46100/50000 rank3 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0252 | Classification_binary: 0.0964 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0448 | Regression_eta: 0.0244 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0379 | LR: 1.13e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.5892s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank3 [2026-10-30 05:23:34,446] INFO : Step 46200/50000 rank3 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0414 | Regression_eta: 0.0302 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0820 | LR: 1.13e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7925s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank3 [2026-10-30 05:28:08,609] INFO : Step 46300/50000 rank3 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0281 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0266 | Regression_energy: 0.0433 | Regression_eta: 0.0235 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0381 | LR: 1.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank3 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank3 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0207 | Classification_binary: 0.0983 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0397 | Regression_eta: 0.0184 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0414 | LR: 1.11e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7892s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank3 [2026-10-30 05:37:21,832] INFO : Step 46500/50000 rank3 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0248 | Classification_binary: 0.1019 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0466 | Regression_eta: 0.0280 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0393 | LR: 1.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8420s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank3 [2026-10-30 05:41:57,344] INFO : Step 46600/50000 rank3 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0463 | Regression_eta: 0.0284 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0621 | LR: 1.10e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7846s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank3 [2026-10-30 05:46:32,412] INFO : Step 46700/50000 rank3 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0447 | Regression_eta: 0.0308 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0401 | LR: 1.10e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6548s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank3 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank3 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0401 | Regression_eta: 0.0251 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0505 | LR: 1.09e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7019s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank3 [2026-10-30 05:55:41,598] INFO : Step 46900/50000 rank3 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0406 | Regression_eta: 0.0350 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0289 | LR: 1.09e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7535s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank3 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank3 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0315 | Regression_energy: 0.0452 | Regression_eta: 0.0322 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0434 | LR: 1.08e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8058s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank3 [2026-10-30 06:00:21,750] INFO : Running validation on rank3 +rank3 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 06:01:12,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 06:01:37,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 06:01:38,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 06:01:39,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 06:01:40,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 06:01:50,472] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:01:51,692] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 06:02:07,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 06:02:08,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 06:02:09,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 06:02:10,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 06:02:20,132] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:02:21,385] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:02:21,870] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 06:02:37,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 06:02:38,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 06:02:39,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 06:02:39,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 06:02:49,901] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 06:07:51,194] INFO : Step 47100/50000 rank3 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0312 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0309 | Regression_eta: 0.0392 | Regression_pt: 0.0312 | Regression_sin_phi: 0.0543 | LR: 1.07e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6252s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank3 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank3 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0372 | Regression_eta: 0.0270 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0467 | LR: 1.07e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9894s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank3 [2026-10-30 06:17:04,150] INFO : Step 47300/50000 rank3 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0345 | Regression_eta: 0.0211 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0502 | LR: 1.06e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0322s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank3 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank3 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.0960 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0344 | Regression_eta: 0.0282 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0440 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4519s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank3 [2026-10-30 06:26:15,800] INFO : Step 47500/50000 rank3 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0485 | Regression_eta: 0.0335 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0357 | LR: 1.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6765s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank3 [2026-10-30 06:30:50,119] INFO : Step 47600/50000 rank3 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0403 | Regression_eta: 0.0399 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0336 | LR: 1.05e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9515s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank3 [2026-10-30 06:35:25,580] INFO : Step 47700/50000 rank3 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0463 | Regression_eta: 0.0426 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0505 | LR: 1.05e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9115s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank3 [2026-10-30 06:40:00,408] INFO : Step 47800/50000 rank3 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0282 | Classification_binary: 0.1037 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0500 | Regression_eta: 0.0228 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0583 | LR: 1.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7803s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank3 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank3 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0425 | Regression_eta: 0.0501 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0465 | LR: 1.04e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8493s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank3 [2026-10-30 06:49:09,636] INFO : Step 48000/50000 rank3 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0541 | Regression_eta: 0.0259 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0408 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8824s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank3 [2026-10-30 06:49:10,726] INFO : Running validation on rank3 +rank3 [2026-10-30 06:49:10,726] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 06:49:10,726] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 06:50:01,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 06:50:27,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 06:50:28,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 06:50:29,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 06:50:30,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 06:50:40,150] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:50:41,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:50:41,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 06:50:57,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 06:50:58,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 06:50:59,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 06:51:00,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 06:51:10,817] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:51:11,430] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 06:51:27,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 06:51:27,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 06:51:28,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 06:51:29,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 06:51:39,565] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank3 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0329 | Regression_eta: 0.0254 | Regression_pt: 0.0344 | Regression_sin_phi: 0.0385 | LR: 1.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6281s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank3 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank3 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1045 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0460 | Regression_eta: 0.0262 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0376 | LR: 1.03e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.3151s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank3 [2026-10-30 07:05:53,465] INFO : Step 48300/50000 rank3 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0330 | Regression_energy: 0.0449 | Regression_eta: 0.0435 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0550 | LR: 1.03e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1161s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank3 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank3 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0496 | Regression_eta: 0.0282 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0400 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8686s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank3 [2026-10-30 07:15:11,422] INFO : Step 48500/50000 rank3 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0335 | Regression_energy: 0.0410 | Regression_eta: 0.0580 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0585 | LR: 1.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.9612s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank3 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank3 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0389 | Regression_eta: 0.0383 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0480 | LR: 1.02e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8077s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank3 [2026-10-30 07:24:30,058] INFO : Step 48700/50000 rank3 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0365 | Regression_eta: 0.0291 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0417 | LR: 1.02e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9967s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank3 [2026-10-30 07:29:10,217] INFO : Step 48800/50000 rank3 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0262 | Classification_binary: 0.0898 | Regression_cos_phi: 0.0298 | Regression_energy: 0.0398 | Regression_eta: 0.0319 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0400 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9226s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank3 [2026-10-30 07:33:48,297] INFO : Step 48900/50000 rank3 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1186 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0525 | Regression_eta: 0.0375 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0606 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7895s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank3 [2026-10-30 07:38:22,689] INFO : Step 49000/50000 rank3 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0367 | Regression_eta: 0.0278 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0572 | LR: 1.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8892s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank3 [2026-10-30 07:38:23,699] INFO : Running validation on rank3 +rank3 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:39:14,632] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 07:39:39,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 07:39:40,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 07:39:41,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 07:39:42,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 07:39:52,289] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 07:39:53,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:39:53,776] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 07:40:09,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 07:40:10,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 07:40:11,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 07:40:12,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 07:40:22,404] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 07:40:23,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:40:24,261] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 07:40:39,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 07:40:40,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 07:40:41,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 07:40:42,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 07:40:52,490] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank3 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0532 | Regression_eta: 0.0248 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0402 | LR: 1.01e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6561s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank3 [2026-10-30 07:50:30,115] INFO : Step 49200/50000 rank3 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0276 | Classification_binary: 0.1023 | Regression_cos_phi: 0.0260 | Regression_energy: 0.0340 | Regression_eta: 0.0359 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0295 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5161s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank3 [2026-10-30 07:55:07,371] INFO : Step 49300/50000 rank3 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0334 | Regression_energy: 0.0574 | Regression_eta: 0.0211 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0358 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8680s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank3 [2026-10-30 07:59:43,350] INFO : Step 49400/50000 rank3 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0391 | Regression_eta: 0.0283 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0389 | LR: 1.00e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8915s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank3 [2026-10-30 08:04:24,759] INFO : Step 49500/50000 rank3 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.0972 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0394 | Regression_eta: 0.0277 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0373 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3356s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank3 [2026-10-30 08:08:58,923] INFO : Step 49600/50000 rank3 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0346 | Regression_eta: 0.0374 | Regression_pt: 0.0356 | Regression_sin_phi: 0.0444 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8759s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank3 [2026-10-30 08:13:36,932] INFO : Step 49700/50000 rank3 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0374 | Regression_eta: 0.0323 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0618 | LR: 1.00e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7051s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank3 [2026-10-30 08:18:11,417] INFO : Step 49800/50000 rank3 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.0973 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0333 | Regression_eta: 0.0251 | Regression_pt: 0.0349 | Regression_sin_phi: 0.0674 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5272s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank3 [2026-10-30 08:22:53,416] INFO : Step 49900/50000 rank3 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1075 | Regression_cos_phi: 0.0319 | Regression_energy: 0.0340 | Regression_eta: 0.0235 | Regression_pt: 0.0356 | Regression_sin_phi: 0.0406 | LR: 1.00e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2025s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank3 [2026-10-30 08:27:37,251] INFO : Step 50000/50000 rank3 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0430 | Regression_eta: 0.0306 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0529 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.3473s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank3 [2026-10-30 08:27:38,186] INFO : Running validation on rank3 +rank3 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 08:28:29,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 08:28:29,586] INFO : Running predictions on clic_edm_ttbar_hits +rank3 [2026-10-30 08:29:00,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 08:29:01,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 08:29:02,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 08:29:03,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 08:29:13,183] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 08:29:13,910] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 08:29:14,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank3 [2026-10-30 08:29:30,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 08:29:32,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 08:29:33,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 08:29:35,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 08:29:45,277] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 08:29:46,349] INFO : Running predictions on clic_edm_qq_hits +rank3 [2026-10-30 08:30:01,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 08:30:02,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 08:30:03,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 08:30:04,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 08:30:14,505] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 08:30:15,232] INFO : Training completed. Total time on device 3: 1059.927min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.4 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..f989d9b8f9cd558ae973df6fb5464deff7be90bf --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.4 @@ -0,0 +1,2654 @@ +rank4 [2026-10-28 11:12:32,422] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-28 11:12:32,422] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank4 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank4 [2026-10-28 11:12:57,209] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-28 11:12:57,209] INFO : Initializing backbone layer 0 +rank4 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank4 [2026-10-28 11:12:57,254] INFO : Initializing backbone layer 2 +rank4 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 3 +rank4 [2026-10-28 11:12:57,272] INFO : Initializing backbone layer 4 +rank4 [2026-10-28 11:12:57,280] INFO : Initializing backbone layer 5 +rank4 [2026-10-28 11:12:57,289] INFO : Backbone initialization took 0.08s +rank4 [2026-10-28 11:12:57,289] INFO : backbone parameters: 7901376 +rank4 [2026-10-28 11:12:57,289] INFO : Initializing output DNNs +rank4 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank4 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank4 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank4 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank4 [2026-10-28 11:12:57,328] INFO : classification_norm parameters: 0 +rank4 [2026-10-28 11:12:57,328] INFO : regression_norm parameters: 0 +rank4 [2026-10-28 11:12:57,328] INFO : classification_query parameters: 0 +rank4 [2026-10-28 11:12:57,328] INFO : regression_query parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : classification_readout parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : regression_readout parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : output_mode=set +rank4 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank4 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank4 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank4 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank4 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=4 +rank4 [2026-10-28 11:12:57,365] INFO : Moved model to device rank=4 +rank4 [2026-10-28 11:12:57,367] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-28 11:12:57,368] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-28 11:12:57,855] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank4 [2026-10-28 11:12:57,855] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-28 11:12:57,856] INFO : Trainable parameters: 12786964 +rank4 [2026-10-28 11:12:57,856] INFO : Non-trainable parameters: 23268 +rank4 [2026-10-28 11:12:57,856] INFO : Total parameters: 12810232 +rank4 [2026-10-28 11:12:57,859] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-28 11:12:57,859] INFO : Getting dataloaders +rank4 [2026-10-28 11:12:57,859] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:03,690] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:05,011] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,001] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-28 11:13:06,001] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-28 11:13:06,001] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,273] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,530] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,793] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-28 11:13:06,793] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-28 11:13:20,633] INFO : Got dataloaders +rank4 [2026-10-28 11:13:20,633] INFO : loader split=train rank=4 len=21108 +rank4 [2026-10-28 11:13:20,633] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-28 11:13:20,634] INFO : Starting training from step 1 to 50000 on rank 4 of 8 +rank4 [2026-10-28 11:13:20,634] INFO : Creating train_iterator +rank4 [2026-10-28 11:13:20,634] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-28 11:13:34,825] INFO : Created train_iterator +rank4 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank4 [2026-10-28 11:18:16,511] INFO : Step 100/50000 rank4 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7163 | Classification_binary: 0.5095 | Regression_cos_phi: 0.0063 | Regression_energy: 0.8672 | Regression_eta: 0.0120 | Regression_pt: 0.8352 | Regression_sin_phi: 0.0067 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.6178s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank4 [2026-10-28 11:22:47,963] INFO : Step 200/50000 rank4 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3674 | Classification_binary: 0.5789 | Regression_cos_phi: 0.4351 | Regression_energy: 0.3668 | Regression_eta: 0.3909 | Regression_pt: 0.3644 | Regression_sin_phi: 0.4086 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 4.4382s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank4 [2026-10-28 11:27:17,891] INFO : Step 300/50000 rank4 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2743 | Classification_binary: 0.4129 | Regression_cos_phi: 0.2340 | Regression_energy: 0.3183 | Regression_eta: 0.2265 | Regression_pt: 0.3242 | Regression_sin_phi: 0.2240 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 4.3670s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank4 [2026-10-28 11:31:48,894] INFO : Step 400/50000 rank4 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2827 | Classification_binary: 0.4659 | Regression_cos_phi: 0.2900 | Regression_energy: 0.3240 | Regression_eta: 0.2590 | Regression_pt: 0.3259 | Regression_sin_phi: 0.2869 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 4.2445s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank4 [2026-10-28 11:36:20,165] INFO : Step 500/50000 rank4 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2520 | Classification_binary: 0.5230 | Regression_cos_phi: 0.3551 | Regression_energy: 0.3196 | Regression_eta: 0.2345 | Regression_pt: 0.3328 | Regression_sin_phi: 0.3255 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.4700s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank4 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank4 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2403 | Classification_binary: 0.4863 | Regression_cos_phi: 0.2652 | Regression_energy: 0.2891 | Regression_eta: 0.2660 | Regression_pt: 0.3015 | Regression_sin_phi: 0.2817 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.5141s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank4 [2026-10-28 11:45:32,573] INFO : Step 700/50000 rank4 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1858 | Classification_binary: 0.3911 | Regression_cos_phi: 0.2253 | Regression_energy: 0.2442 | Regression_eta: 0.1619 | Regression_pt: 0.2514 | Regression_sin_phi: 0.1968 | LR: 1.00e-03 | DataLoad Time: 0.0177s | Model Forward Time: 3.7408s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank4 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank4 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2346 | Classification_binary: 0.4751 | Regression_cos_phi: 0.2307 | Regression_energy: 0.2780 | Regression_eta: 0.2258 | Regression_pt: 0.2859 | Regression_sin_phi: 0.2225 | LR: 9.99e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.2805s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank4 [2026-10-28 11:54:58,535] INFO : Step 900/50000 rank4 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2696 | Classification_binary: 0.4986 | Regression_cos_phi: 0.2963 | Regression_energy: 0.2740 | Regression_eta: 0.3104 | Regression_pt: 0.2866 | Regression_sin_phi: 0.3786 | LR: 9.99e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4552s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank4 [2026-10-28 11:59:55,232] INFO : Step 1000/50000 rank4 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1996 | Classification_binary: 0.3803 | Regression_cos_phi: 0.1441 | Regression_energy: 0.2181 | Regression_eta: 0.1603 | Regression_pt: 0.2273 | Regression_sin_phi: 0.1788 | LR: 9.99e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.7398s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank4 [2026-10-28 11:59:56,332] INFO : Running validation on rank4 +rank4 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 12:00:48,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:00:49,300] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 12:01:16,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:01:17,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:01:19,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:01:20,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:01:30,050] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:01:30,839] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 12:01:46,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:01:47,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:01:48,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:01:49,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:01:59,866] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 12:02:00,465] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:02:01,008] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 12:02:16,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:02:17,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:02:18,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:02:18,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:02:29,002] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:07:33,552] INFO : Step 1100/50000 rank4 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2774 | Classification_binary: 0.4640 | Regression_cos_phi: 0.2617 | Regression_energy: 0.2953 | Regression_eta: 0.2079 | Regression_pt: 0.3063 | Regression_sin_phi: 0.2169 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8140s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank4 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank4 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2228 | Classification_binary: 0.4839 | Regression_cos_phi: 0.3297 | Regression_energy: 0.2591 | Regression_eta: 0.1896 | Regression_pt: 0.2727 | Regression_sin_phi: 0.2836 | LR: 9.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4361s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank4 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank4 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1822 | Classification_binary: 0.3649 | Regression_cos_phi: 0.2340 | Regression_energy: 0.2389 | Regression_eta: 0.1470 | Regression_pt: 0.2472 | Regression_sin_phi: 0.1667 | LR: 9.98e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2253s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank4 [2026-10-28 12:21:55,310] INFO : Step 1400/50000 rank4 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2042 | Classification_binary: 0.4260 | Regression_cos_phi: 0.1864 | Regression_energy: 0.2506 | Regression_eta: 0.1859 | Regression_pt: 0.2622 | Regression_sin_phi: 0.2830 | LR: 9.98e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5559s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank4 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank4 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1810 | Classification_binary: 0.4515 | Regression_cos_phi: 0.1767 | Regression_energy: 0.2542 | Regression_eta: 0.1730 | Regression_pt: 0.2603 | Regression_sin_phi: 0.1998 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9965s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank4 [2026-10-28 12:31:13,938] INFO : Step 1600/50000 rank4 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1644 | Classification_binary: 0.4408 | Regression_cos_phi: 0.2248 | Regression_energy: 0.2269 | Regression_eta: 0.1487 | Regression_pt: 0.2348 | Regression_sin_phi: 0.1995 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8069s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank4 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank4 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1518 | Classification_binary: 0.3267 | Regression_cos_phi: 0.1774 | Regression_energy: 0.2629 | Regression_eta: 0.2432 | Regression_pt: 0.2694 | Regression_sin_phi: 0.1847 | LR: 9.97e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank4 [2026-10-28 12:40:54,251] INFO : Step 1800/50000 rank4 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1964 | Classification_binary: 0.3963 | Regression_cos_phi: 0.1738 | Regression_energy: 0.2671 | Regression_eta: 0.1592 | Regression_pt: 0.2783 | Regression_sin_phi: 0.2120 | LR: 9.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6783s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank4 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank4 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2334 | Classification_binary: 0.4731 | Regression_cos_phi: 0.2823 | Regression_energy: 0.2261 | Regression_eta: 0.2196 | Regression_pt: 0.2356 | Regression_sin_phi: 0.2578 | LR: 9.97e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4299s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank4 [2026-10-28 12:50:22,599] INFO : Step 2000/50000 rank4 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1268 | Classification_binary: 0.3579 | Regression_cos_phi: 0.2072 | Regression_energy: 0.2179 | Regression_eta: 0.1322 | Regression_pt: 0.2230 | Regression_sin_phi: 0.1499 | LR: 9.96e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9893s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank4 [2026-10-28 12:50:23,688] INFO : Running validation on rank4 +rank4 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 12:51:19,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:51:20,335] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 12:51:47,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:51:48,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:51:49,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:51:51,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:52:01,155] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 12:52:18,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:52:19,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:52:20,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:52:21,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:52:31,529] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 12:52:31,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:52:32,092] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 12:52:47,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:52:48,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:52:49,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:52:50,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:53:00,781] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank4 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1743 | Classification_binary: 0.3868 | Regression_cos_phi: 0.1779 | Regression_energy: 0.2071 | Regression_eta: 0.1515 | Regression_pt: 0.2202 | Regression_sin_phi: 0.2226 | LR: 9.96e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6446s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank4 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank4 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2177 | Classification_binary: 0.4465 | Regression_cos_phi: 0.2233 | Regression_energy: 0.2496 | Regression_eta: 0.2097 | Regression_pt: 0.2597 | Regression_sin_phi: 0.2110 | LR: 9.96e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.4352s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank4 [2026-10-28 13:08:12,323] INFO : Step 2300/50000 rank4 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1498 | Classification_binary: 0.3857 | Regression_cos_phi: 0.1922 | Regression_energy: 0.2231 | Regression_eta: 0.1555 | Regression_pt: 0.2316 | Regression_sin_phi: 0.1731 | LR: 9.95e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.1083s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank4 [2026-10-28 13:13:00,652] INFO : Step 2400/50000 rank4 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1434 | Classification_binary: 0.3510 | Regression_cos_phi: 0.2347 | Regression_energy: 0.2038 | Regression_eta: 0.1377 | Regression_pt: 0.2097 | Regression_sin_phi: 0.1374 | LR: 9.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2191s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank4 [2026-10-28 13:18:06,886] INFO : Step 2500/50000 rank4 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1933 | Classification_binary: 0.3902 | Regression_cos_phi: 0.2665 | Regression_energy: 0.2101 | Regression_eta: 0.1476 | Regression_pt: 0.2197 | Regression_sin_phi: 0.2841 | LR: 9.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.0083s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank4 [2026-10-28 13:22:57,737] INFO : Step 2600/50000 rank4 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1473 | Classification_binary: 0.4343 | Regression_cos_phi: 0.2050 | Regression_energy: 0.2502 | Regression_eta: 0.1642 | Regression_pt: 0.2609 | Regression_sin_phi: 0.2334 | LR: 9.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3783s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank4 [2026-10-28 13:27:39,814] INFO : Step 2700/50000 rank4 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1237 | Classification_binary: 0.3373 | Regression_cos_phi: 0.1709 | Regression_energy: 0.1883 | Regression_eta: 0.1263 | Regression_pt: 0.1919 | Regression_sin_phi: 0.1774 | LR: 9.94e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5221s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank4 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank4 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1686 | Classification_binary: 0.3729 | Regression_cos_phi: 0.1950 | Regression_energy: 0.2160 | Regression_eta: 0.1497 | Regression_pt: 0.2245 | Regression_sin_phi: 0.2300 | LR: 9.93e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2928s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank4 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank4 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1768 | Classification_binary: 0.3998 | Regression_cos_phi: 0.2386 | Regression_energy: 0.2282 | Regression_eta: 0.1681 | Regression_pt: 0.2370 | Regression_sin_phi: 0.2509 | LR: 9.93e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5103s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank4 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank4 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1347 | Classification_binary: 0.3411 | Regression_cos_phi: 0.2098 | Regression_energy: 0.1764 | Regression_eta: 0.1326 | Regression_pt: 0.1817 | Regression_sin_phi: 0.1721 | LR: 9.92e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5883s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank4 [2026-10-28 13:42:04,672] INFO : Running validation on rank4 +rank4 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:42:57,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:42:58,471] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 13:43:25,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:43:26,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:43:27,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:43:28,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:43:38,345] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:43:39,344] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 13:43:55,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:43:56,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:43:57,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:43:58,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:44:08,522] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:44:09,100] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 13:44:24,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:44:25,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:44:26,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:44:26,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:44:36,970] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 13:49:29,794] INFO : Step 3100/50000 rank4 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1887 | Classification_binary: 0.3445 | Regression_cos_phi: 0.2191 | Regression_energy: 0.2121 | Regression_eta: 0.1485 | Regression_pt: 0.2197 | Regression_sin_phi: 0.2312 | LR: 9.91e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8440s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank4 [2026-10-28 13:54:22,657] INFO : Step 3200/50000 rank4 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1723 | Classification_binary: 0.4501 | Regression_cos_phi: 0.3145 | Regression_energy: 0.2203 | Regression_eta: 0.2006 | Regression_pt: 0.2307 | Regression_sin_phi: 0.3040 | LR: 9.91e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.0550s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank4 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank4 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1373 | Classification_binary: 0.3561 | Regression_cos_phi: 0.1741 | Regression_energy: 0.2010 | Regression_eta: 0.1420 | Regression_pt: 0.2071 | Regression_sin_phi: 0.1848 | LR: 9.90e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7404s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank4 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank4 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1869 | Classification_binary: 0.3213 | Regression_cos_phi: 0.2353 | Regression_energy: 0.2415 | Regression_eta: 0.1890 | Regression_pt: 0.2428 | Regression_sin_phi: 0.1922 | LR: 9.90e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9447s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank4 [2026-10-28 14:08:16,395] INFO : Step 3500/50000 rank4 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1473 | Classification_binary: 0.3427 | Regression_cos_phi: 0.1630 | Regression_energy: 0.1863 | Regression_eta: 0.1094 | Regression_pt: 0.1925 | Regression_sin_phi: 0.1593 | LR: 9.89e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7454s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank4 [2026-10-28 14:13:02,151] INFO : Step 3600/50000 rank4 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 0.3872 | Regression_cos_phi: 0.1652 | Regression_energy: 0.1951 | Regression_eta: 0.1329 | Regression_pt: 0.2035 | Regression_sin_phi: 0.1938 | LR: 9.89e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8788s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank4 [2026-10-28 14:17:42,403] INFO : Step 3700/50000 rank4 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1241 | Classification_binary: 0.3287 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1796 | Regression_eta: 0.1239 | Regression_pt: 0.1864 | Regression_sin_phi: 0.1981 | LR: 9.88e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2566s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank4 [2026-10-28 14:22:44,343] INFO : Step 3800/50000 rank4 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1552 | Classification_binary: 0.3600 | Regression_cos_phi: 0.2339 | Regression_energy: 0.2275 | Regression_eta: 0.1542 | Regression_pt: 0.2363 | Regression_sin_phi: 0.2182 | LR: 9.87e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1824s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank4 [2026-10-28 14:27:33,639] INFO : Step 3900/50000 rank4 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1647 | Classification_binary: 0.3829 | Regression_cos_phi: 0.2183 | Regression_energy: 0.2166 | Regression_eta: 0.1376 | Regression_pt: 0.2251 | Regression_sin_phi: 0.2212 | LR: 9.87e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.8171s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank4 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank4 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1004 | Classification_binary: 0.3296 | Regression_cos_phi: 0.1757 | Regression_energy: 0.2325 | Regression_eta: 0.0958 | Regression_pt: 0.2414 | Regression_sin_phi: 0.1475 | LR: 9.86e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5865s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank4 [2026-10-28 14:32:39,608] INFO : Running validation on rank4 +rank4 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 14:33:34,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:33:34,720] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 14:34:03,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 14:34:05,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 14:34:09,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 14:34:11,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 14:34:21,237] INFO : Time taken to make predictions on device 4 is: 0.35 min +rank4 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:34:24,285] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 14:34:40,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 14:34:42,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 14:34:43,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 14:34:44,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 14:34:54,945] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:34:56,346] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 14:35:12,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 14:35:12,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 14:35:13,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 14:35:14,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 14:35:24,559] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank4 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1191 | Classification_binary: 0.3391 | Regression_cos_phi: 0.1676 | Regression_energy: 0.1824 | Regression_eta: 0.1314 | Regression_pt: 0.1892 | Regression_sin_phi: 0.2373 | LR: 9.85e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8514s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank4 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank4 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1611 | Classification_binary: 0.3623 | Regression_cos_phi: 0.2020 | Regression_energy: 0.2166 | Regression_eta: 0.1388 | Regression_pt: 0.2220 | Regression_sin_phi: 0.1763 | LR: 9.84e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5083s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank4 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank4 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1586 | Classification_binary: 0.3823 | Regression_cos_phi: 0.2191 | Regression_energy: 0.2098 | Regression_eta: 0.1582 | Regression_pt: 0.2198 | Regression_sin_phi: 0.2719 | LR: 9.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4595s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank4 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank4 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1355 | Classification_binary: 0.2993 | Regression_cos_phi: 0.1562 | Regression_energy: 0.2189 | Regression_eta: 0.1201 | Regression_pt: 0.2257 | Regression_sin_phi: 0.1588 | LR: 9.83e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.1734s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank4 [2026-10-28 14:59:28,687] INFO : Step 4500/50000 rank4 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1452 | Classification_binary: 0.3911 | Regression_cos_phi: 0.2039 | Regression_energy: 0.1838 | Regression_eta: 0.1446 | Regression_pt: 0.1933 | Regression_sin_phi: 0.2245 | LR: 9.82e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2514s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank4 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank4 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1315 | Classification_binary: 0.4236 | Regression_cos_phi: 0.1946 | Regression_energy: 0.1685 | Regression_eta: 0.1364 | Regression_pt: 0.1755 | Regression_sin_phi: 0.1550 | LR: 9.81e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7016s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank4 [2026-10-28 15:09:18,561] INFO : Step 4700/50000 rank4 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.3131 | Regression_cos_phi: 0.1312 | Regression_energy: 0.1677 | Regression_eta: 0.0985 | Regression_pt: 0.1752 | Regression_sin_phi: 0.1249 | LR: 9.81e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7053s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank4 [2026-10-28 15:14:30,754] INFO : Step 4800/50000 rank4 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1586 | Classification_binary: 0.3346 | Regression_cos_phi: 0.2139 | Regression_energy: 0.1833 | Regression_eta: 0.1537 | Regression_pt: 0.1908 | Regression_sin_phi: 0.1822 | LR: 9.80e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4406s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank4 [2026-10-28 15:19:27,571] INFO : Step 4900/50000 rank4 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1706 | Classification_binary: 0.4515 | Regression_cos_phi: 0.2392 | Regression_energy: 0.1794 | Regression_eta: 0.1584 | Regression_pt: 0.1855 | Regression_sin_phi: 0.2110 | LR: 9.79e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5348s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank4 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank4 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1238 | Classification_binary: 0.2925 | Regression_cos_phi: 0.1646 | Regression_energy: 0.1799 | Regression_eta: 0.1224 | Regression_pt: 0.1851 | Regression_sin_phi: 0.1023 | LR: 9.78e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4815s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank4 [2026-10-28 15:24:24,982] INFO : Running validation on rank4 +rank4 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 15:25:19,075] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:25:19,571] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 15:25:44,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 15:25:46,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 15:25:47,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 15:25:48,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 15:25:58,122] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:25:59,251] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 15:26:15,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 15:26:16,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 15:26:17,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 15:26:18,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 15:26:28,602] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:26:28,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 15:26:45,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 15:26:46,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 15:26:46,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 15:26:47,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 15:26:57,797] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 15:32:01,930] INFO : Step 5100/50000 rank4 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1106 | Classification_binary: 0.2544 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1968 | Regression_eta: 0.1201 | Regression_pt: 0.2020 | Regression_sin_phi: 0.1653 | LR: 9.77e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.6407s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank4 [2026-10-28 15:36:53,511] INFO : Step 5200/50000 rank4 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1314 | Classification_binary: 0.3289 | Regression_cos_phi: 0.1622 | Regression_energy: 0.1894 | Regression_eta: 0.1188 | Regression_pt: 0.1965 | Regression_sin_phi: 0.1764 | LR: 9.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.6372s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank4 [2026-10-28 15:41:36,526] INFO : Step 5300/50000 rank4 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1385 | Classification_binary: 0.3307 | Regression_cos_phi: 0.1665 | Regression_energy: 0.1889 | Regression_eta: 0.1060 | Regression_pt: 0.1952 | Regression_sin_phi: 0.1659 | LR: 9.75e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.9269s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank4 [2026-10-28 15:46:33,435] INFO : Step 5400/50000 rank4 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1176 | Classification_binary: 0.3336 | Regression_cos_phi: 0.1830 | Regression_energy: 0.1896 | Regression_eta: 0.1081 | Regression_pt: 0.1920 | Regression_sin_phi: 0.1960 | LR: 9.74e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4635s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank4 [2026-10-28 15:51:26,878] INFO : Step 5500/50000 rank4 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1388 | Classification_binary: 0.3347 | Regression_cos_phi: 0.2242 | Regression_energy: 0.1840 | Regression_eta: 0.1402 | Regression_pt: 0.1919 | Regression_sin_phi: 0.1282 | LR: 9.73e-04 | DataLoad Time: 0.0190s | Model Forward Time: 6.5350s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank4 [2026-10-28 15:56:25,712] INFO : Step 5600/50000 rank4 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1403 | Classification_binary: 0.4206 | Regression_cos_phi: 0.1965 | Regression_energy: 0.1755 | Regression_eta: 0.1471 | Regression_pt: 0.1819 | Regression_sin_phi: 0.2341 | LR: 9.72e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.3187s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank4 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank4 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.2866 | Regression_cos_phi: 0.1575 | Regression_energy: 0.1623 | Regression_eta: 0.0895 | Regression_pt: 0.1639 | Regression_sin_phi: 0.1452 | LR: 9.71e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5512s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank4 [2026-10-28 16:06:09,459] INFO : Step 5800/50000 rank4 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1756 | Classification_binary: 0.3638 | Regression_cos_phi: 0.1892 | Regression_energy: 0.1750 | Regression_eta: 0.1649 | Regression_pt: 0.1833 | Regression_sin_phi: 0.2424 | LR: 9.70e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.2269s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank4 [2026-10-28 16:11:19,901] INFO : Step 5900/50000 rank4 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1234 | Classification_binary: 0.3670 | Regression_cos_phi: 0.2227 | Regression_energy: 0.1581 | Regression_eta: 0.1380 | Regression_pt: 0.1634 | Regression_sin_phi: 0.2074 | LR: 9.69e-04 | DataLoad Time: 0.0163s | Model Forward Time: 7.1477s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank4 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank4 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1506 | Classification_binary: 0.3574 | Regression_cos_phi: 0.2335 | Regression_energy: 0.1818 | Regression_eta: 0.1504 | Regression_pt: 0.1885 | Regression_sin_phi: 0.1732 | LR: 9.68e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4971s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank4 [2026-10-28 16:16:41,624] INFO : Running validation on rank4 +rank4 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:17:32,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:17:33,086] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 16:17:59,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:18:00,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:18:01,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:18:02,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:18:12,645] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:18:13,873] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 16:18:29,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:18:31,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:18:31,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:18:32,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:18:42,958] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:18:43,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:18:43,953] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 16:18:59,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:19:00,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:19:02,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:19:03,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:19:13,768] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank4 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1065 | Classification_binary: 0.2066 | Regression_cos_phi: 0.1138 | Regression_energy: 0.1901 | Regression_eta: 0.0814 | Regression_pt: 0.1944 | Regression_sin_phi: 0.1564 | LR: 9.67e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9557s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank4 [2026-10-28 16:29:26,889] INFO : Step 6200/50000 rank4 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.3289 | Regression_cos_phi: 0.1872 | Regression_energy: 0.1838 | Regression_eta: 0.1164 | Regression_pt: 0.1932 | Regression_sin_phi: 0.1981 | LR: 9.66e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0898s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank4 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank4 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1529 | Classification_binary: 0.3332 | Regression_cos_phi: 0.2078 | Regression_energy: 0.1817 | Regression_eta: 0.1445 | Regression_pt: 0.1883 | Regression_sin_phi: 0.1802 | LR: 9.65e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0297s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank4 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank4 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.2835 | Regression_cos_phi: 0.1561 | Regression_energy: 0.1417 | Regression_eta: 0.0990 | Regression_pt: 0.1449 | Regression_sin_phi: 0.1776 | LR: 9.64e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7770s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank4 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank4 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1340 | Classification_binary: 0.2847 | Regression_cos_phi: 0.1486 | Regression_energy: 0.1519 | Regression_eta: 0.1172 | Regression_pt: 0.1593 | Regression_sin_phi: 0.1937 | LR: 9.63e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5007s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank4 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank4 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.4335 | Regression_cos_phi: 0.2603 | Regression_energy: 0.1587 | Regression_eta: 0.2136 | Regression_pt: 0.1608 | Regression_sin_phi: 0.2326 | LR: 9.62e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8072s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank4 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank4 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1190 | Classification_binary: 0.3054 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1686 | Regression_eta: 0.1006 | Regression_pt: 0.1723 | Regression_sin_phi: 0.1261 | LR: 9.61e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.3367s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank4 [2026-10-28 16:58:34,483] INFO : Step 6800/50000 rank4 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1324 | Classification_binary: 0.2623 | Regression_cos_phi: 0.1202 | Regression_energy: 0.1745 | Regression_eta: 0.1160 | Regression_pt: 0.1799 | Regression_sin_phi: 0.1773 | LR: 9.60e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.4641s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank4 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank4 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3281 | Regression_cos_phi: 0.1761 | Regression_energy: 0.1560 | Regression_eta: 0.1330 | Regression_pt: 0.1605 | Regression_sin_phi: 0.1995 | LR: 9.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6896s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank4 [2026-10-28 17:08:29,916] INFO : Step 7000/50000 rank4 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1252 | Classification_binary: 0.3669 | Regression_cos_phi: 0.1749 | Regression_energy: 0.1630 | Regression_eta: 0.1285 | Regression_pt: 0.1647 | Regression_sin_phi: 0.1577 | LR: 9.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9238s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank4 [2026-10-28 17:08:30,791] INFO : Running validation on rank4 +rank4 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 17:09:23,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:09:24,267] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 17:09:51,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 17:09:53,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 17:09:55,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 17:09:56,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 17:10:06,926] INFO : Time taken to make predictions on device 4 is: 0.29 min +rank4 [2026-10-28 17:10:07,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:10:08,261] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 17:10:24,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 17:10:25,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 17:10:26,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 17:10:27,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 17:10:37,146] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:10:37,866] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:10:38,448] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 17:10:54,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 17:10:55,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 17:10:55,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 17:10:56,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 17:11:06,822] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 17:16:27,594] INFO : Step 7100/50000 rank4 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0934 | Classification_binary: 0.2543 | Regression_cos_phi: 0.1356 | Regression_energy: 0.1517 | Regression_eta: 0.0780 | Regression_pt: 0.1582 | Regression_sin_phi: 0.1522 | LR: 9.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1466s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank4 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank4 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1243 | Classification_binary: 0.3135 | Regression_cos_phi: 0.1762 | Regression_energy: 0.1595 | Regression_eta: 0.1204 | Regression_pt: 0.1649 | Regression_sin_phi: 0.1685 | LR: 9.55e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3456s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank4 [2026-10-28 17:26:34,943] INFO : Step 7300/50000 rank4 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1168 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1448 | Regression_energy: 0.1597 | Regression_eta: 0.1381 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1360 | LR: 9.53e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4027s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank4 [2026-10-28 17:31:34,684] INFO : Step 7400/50000 rank4 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0956 | Classification_binary: 0.2747 | Regression_cos_phi: 0.1471 | Regression_energy: 0.1431 | Regression_eta: 0.0858 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0943 | LR: 9.52e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5942s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank4 [2026-10-28 17:36:30,884] INFO : Step 7500/50000 rank4 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2569 | Regression_cos_phi: 0.1550 | Regression_energy: 0.1506 | Regression_eta: 0.0845 | Regression_pt: 0.1578 | Regression_sin_phi: 0.1235 | LR: 9.51e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6593s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank4 [2026-10-28 17:41:09,113] INFO : Step 7600/50000 rank4 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1217 | Classification_binary: 0.3463 | Regression_cos_phi: 0.1654 | Regression_energy: 0.1878 | Regression_eta: 0.1296 | Regression_pt: 0.1955 | Regression_sin_phi: 0.1672 | LR: 9.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7898s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank4 [2026-10-28 17:46:03,243] INFO : Step 7700/50000 rank4 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1011 | Classification_binary: 0.3060 | Regression_cos_phi: 0.1372 | Regression_energy: 0.1467 | Regression_eta: 0.0897 | Regression_pt: 0.1494 | Regression_sin_phi: 0.1304 | LR: 9.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7829s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank4 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank4 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0970 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1131 | Regression_energy: 0.1522 | Regression_eta: 0.0800 | Regression_pt: 0.1522 | Regression_sin_phi: 0.1060 | LR: 9.47e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4013s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank4 [2026-10-28 17:56:05,209] INFO : Step 7900/50000 rank4 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1222 | Classification_binary: 0.3320 | Regression_cos_phi: 0.1243 | Regression_energy: 0.1380 | Regression_eta: 0.1152 | Regression_pt: 0.1432 | Regression_sin_phi: 0.1363 | LR: 9.46e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.3975s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank4 [2026-10-28 18:00:55,277] INFO : Step 8000/50000 rank4 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1600 | Classification_binary: 0.3181 | Regression_cos_phi: 0.1715 | Regression_energy: 0.1575 | Regression_eta: 0.1787 | Regression_pt: 0.1611 | Regression_sin_phi: 0.1863 | LR: 9.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4814s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank4 [2026-10-28 18:00:56,228] INFO : Running validation on rank4 +rank4 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:01:47,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:01:48,315] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 18:02:14,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:02:15,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:02:16,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:02:17,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:02:27,846] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:02:28,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 18:02:45,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:02:46,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:02:47,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:02:48,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:02:58,055] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:02:58,280] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:02:58,745] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 18:03:14,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:03:14,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:03:15,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:03:16,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:03:26,595] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 18:08:30,022] INFO : Step 8100/50000 rank4 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2487 | Regression_cos_phi: 0.0862 | Regression_energy: 0.1448 | Regression_eta: 0.0769 | Regression_pt: 0.1488 | Regression_sin_phi: 0.1242 | LR: 9.43e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.3416s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank4 [2026-10-28 18:13:41,921] INFO : Step 8200/50000 rank4 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.2857 | Regression_cos_phi: 0.1826 | Regression_energy: 0.1253 | Regression_eta: 0.1260 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1940 | LR: 9.42e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3253s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank4 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank4 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1213 | Classification_binary: 0.3492 | Regression_cos_phi: 0.1524 | Regression_energy: 0.1411 | Regression_eta: 0.1106 | Regression_pt: 0.1458 | Regression_sin_phi: 0.1805 | LR: 9.40e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6369s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank4 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank4 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0906 | Classification_binary: 0.2379 | Regression_cos_phi: 0.1198 | Regression_energy: 0.1316 | Regression_eta: 0.1174 | Regression_pt: 0.1385 | Regression_sin_phi: 0.1139 | LR: 9.39e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.1825s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank4 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank4 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1018 | Classification_binary: 0.2789 | Regression_cos_phi: 0.1509 | Regression_energy: 0.1333 | Regression_eta: 0.0997 | Regression_pt: 0.1377 | Regression_sin_phi: 0.1830 | LR: 9.37e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5046s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank4 [2026-10-28 18:34:09,472] INFO : Step 8600/50000 rank4 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1016 | Classification_binary: 0.3196 | Regression_cos_phi: 0.1312 | Regression_energy: 0.1464 | Regression_eta: 0.0866 | Regression_pt: 0.1525 | Regression_sin_phi: 0.1513 | LR: 9.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7741s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank4 [2026-10-28 18:39:14,579] INFO : Step 8700/50000 rank4 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1044 | Classification_binary: 0.2856 | Regression_cos_phi: 0.1441 | Regression_energy: 0.1272 | Regression_eta: 0.0857 | Regression_pt: 0.1279 | Regression_sin_phi: 0.1442 | LR: 9.34e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.7195s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank4 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank4 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0991 | Regression_energy: 0.1171 | Regression_eta: 0.0703 | Regression_pt: 0.1199 | Regression_sin_phi: 0.1151 | LR: 9.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.4108s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank4 [2026-10-28 18:49:28,517] INFO : Step 8900/50000 rank4 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1259 | Classification_binary: 0.3398 | Regression_cos_phi: 0.1906 | Regression_energy: 0.1317 | Regression_eta: 0.1160 | Regression_pt: 0.1389 | Regression_sin_phi: 0.1884 | LR: 9.31e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7927s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank4 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank4 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.3269 | Regression_cos_phi: 0.1487 | Regression_energy: 0.1293 | Regression_eta: 0.1073 | Regression_pt: 0.1354 | Regression_sin_phi: 0.1151 | LR: 9.30e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8402s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank4 [2026-10-28 18:54:54,936] INFO : Running validation on rank4 +rank4 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:55:52,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:55:53,257] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 18:56:19,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:56:21,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:56:22,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:56:23,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:56:33,840] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:56:35,473] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 18:56:51,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:56:52,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:56:53,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:56:54,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:57:04,366] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:57:05,965] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 18:57:21,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:57:22,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:57:23,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:57:23,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:57:33,930] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 19:03:08,597] INFO : Step 9100/50000 rank4 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.2552 | Regression_cos_phi: 0.1021 | Regression_energy: 0.1121 | Regression_eta: 0.0972 | Regression_pt: 0.1158 | Regression_sin_phi: 0.1264 | LR: 9.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.1689s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank4 [2026-10-28 19:08:18,223] INFO : Step 9200/50000 rank4 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2758 | Regression_cos_phi: 0.1316 | Regression_energy: 0.1222 | Regression_eta: 0.0836 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0994 | LR: 9.27e-04 | DataLoad Time: 0.0243s | Model Forward Time: 6.5447s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank4 [2026-10-28 19:13:30,253] INFO : Step 9300/50000 rank4 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1039 | Classification_binary: 0.3230 | Regression_cos_phi: 0.1565 | Regression_energy: 0.1317 | Regression_eta: 0.1011 | Regression_pt: 0.1368 | Regression_sin_phi: 0.1084 | LR: 9.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3902s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank4 [2026-10-28 19:18:55,614] INFO : Step 9400/50000 rank4 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0855 | Classification_binary: 0.2635 | Regression_cos_phi: 0.1203 | Regression_energy: 0.1231 | Regression_eta: 0.0990 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1836 | LR: 9.24e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2021s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank4 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank4 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2162 | Regression_cos_phi: 0.1187 | Regression_energy: 0.1361 | Regression_eta: 0.0919 | Regression_pt: 0.1402 | Regression_sin_phi: 0.1188 | LR: 9.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2842s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank4 [2026-10-28 19:29:34,439] INFO : Step 9600/50000 rank4 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1085 | Classification_binary: 0.3260 | Regression_cos_phi: 0.1665 | Regression_energy: 0.1277 | Regression_eta: 0.1108 | Regression_pt: 0.1382 | Regression_sin_phi: 0.1499 | LR: 9.21e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6041s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank4 [2026-10-28 19:34:53,791] INFO : Step 9700/50000 rank4 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.3141 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1384 | Regression_eta: 0.0946 | Regression_pt: 0.1437 | Regression_sin_phi: 0.1398 | LR: 9.19e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank4 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank4 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2619 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1264 | Regression_eta: 0.0859 | Regression_pt: 0.1312 | Regression_sin_phi: 0.1540 | LR: 9.17e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5632s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank4 [2026-10-28 19:44:46,547] INFO : Step 9900/50000 rank4 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0966 | Regression_energy: 0.1176 | Regression_eta: 0.0738 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0781 | LR: 9.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9300s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank4 [2026-10-28 19:49:37,631] INFO : Step 10000/50000 rank4 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1138 | Classification_binary: 0.3392 | Regression_cos_phi: 0.1608 | Regression_energy: 0.1282 | Regression_eta: 0.1053 | Regression_pt: 0.1348 | Regression_sin_phi: 0.1339 | LR: 9.14e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.6254s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank4 [2026-10-28 19:49:38,690] INFO : Running validation on rank4 +rank4 [2026-10-28 19:49:38,690] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 19:50:35,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:50:35,625] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 19:51:01,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 19:51:02,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 19:51:03,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 19:51:05,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 19:51:15,099] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 19:51:15,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:51:16,384] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 19:51:32,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 19:51:33,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 19:51:34,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 19:51:35,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 19:51:45,519] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:51:46,146] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:51:46,687] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 19:52:02,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 19:52:03,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 19:52:04,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 19:52:05,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 19:52:15,150] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 19:57:27,968] INFO : Step 10100/50000 rank4 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0960 | Regression_energy: 0.1400 | Regression_eta: 0.0785 | Regression_pt: 0.1464 | Regression_sin_phi: 0.1224 | LR: 9.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7460s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank4 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank4 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0883 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0891 | Regression_energy: 0.1296 | Regression_eta: 0.0899 | Regression_pt: 0.1306 | Regression_sin_phi: 0.1297 | LR: 9.11e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6809s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank4 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank4 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2755 | Regression_cos_phi: 0.0998 | Regression_energy: 0.1167 | Regression_eta: 0.0755 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0951 | LR: 9.09e-04 | DataLoad Time: 0.0230s | Model Forward Time: 5.1993s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank4 [2026-10-28 20:12:49,147] INFO : Step 10400/50000 rank4 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.2688 | Regression_cos_phi: 0.1210 | Regression_energy: 0.1237 | Regression_eta: 0.0740 | Regression_pt: 0.1280 | Regression_sin_phi: 0.1203 | LR: 9.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2215s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank4 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank4 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.1947 | Regression_cos_phi: 0.1561 | Regression_energy: 0.1107 | Regression_eta: 0.0973 | Regression_pt: 0.1121 | Regression_sin_phi: 0.1334 | LR: 9.06e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3227s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank4 [2026-10-28 20:23:17,055] INFO : Step 10600/50000 rank4 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.3000 | Regression_cos_phi: 0.1166 | Regression_energy: 0.1240 | Regression_eta: 0.0822 | Regression_pt: 0.1280 | Regression_sin_phi: 0.1129 | LR: 9.04e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0921s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank4 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank4 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2953 | Regression_cos_phi: 0.1241 | Regression_energy: 0.1143 | Regression_eta: 0.0774 | Regression_pt: 0.1168 | Regression_sin_phi: 0.1283 | LR: 9.02e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8719s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank4 [2026-10-28 20:33:28,996] INFO : Step 10800/50000 rank4 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0851 | Regression_energy: 0.1124 | Regression_eta: 0.0718 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0787 | LR: 9.00e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.6816s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank4 [2026-10-28 20:38:44,060] INFO : Step 10900/50000 rank4 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2269 | Regression_cos_phi: 0.1138 | Regression_energy: 0.0934 | Regression_eta: 0.0757 | Regression_pt: 0.0968 | Regression_sin_phi: 0.1035 | LR: 8.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7053s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank4 [2026-10-28 20:43:46,527] INFO : Step 11000/50000 rank4 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.3234 | Regression_cos_phi: 0.1347 | Regression_energy: 0.1143 | Regression_eta: 0.0884 | Regression_pt: 0.1202 | Regression_sin_phi: 0.1273 | LR: 8.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4443s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank4 [2026-10-28 20:43:47,416] INFO : Running validation on rank4 +rank4 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 20:44:41,298] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:44:41,886] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 20:45:09,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 20:45:10,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 20:45:12,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 20:45:13,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 20:45:23,086] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 20:45:23,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:45:24,125] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 20:45:40,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 20:45:41,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 20:45:42,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 20:45:44,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 20:45:54,032] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 20:46:11,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 20:46:12,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 20:46:14,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 20:46:15,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 20:46:25,462] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 20:51:55,950] INFO : Step 11100/50000 rank4 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1392 | Regression_energy: 0.1185 | Regression_eta: 0.0772 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0967 | LR: 8.95e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8320s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank4 [2026-10-28 20:57:10,090] INFO : Step 11200/50000 rank4 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.2212 | Regression_cos_phi: 0.1212 | Regression_energy: 0.1024 | Regression_eta: 0.0713 | Regression_pt: 0.1060 | Regression_sin_phi: 0.1090 | LR: 8.93e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank4 [2026-10-28 21:02:18,460] INFO : Step 11300/50000 rank4 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2466 | Regression_cos_phi: 0.1303 | Regression_energy: 0.1112 | Regression_eta: 0.0748 | Regression_pt: 0.1150 | Regression_sin_phi: 0.1360 | LR: 8.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2918s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank4 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank4 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2744 | Regression_cos_phi: 0.1084 | Regression_energy: 0.1091 | Regression_eta: 0.0677 | Regression_pt: 0.1136 | Regression_sin_phi: 0.1087 | LR: 8.89e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9852s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank4 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank4 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1982 | Regression_cos_phi: 0.1005 | Regression_energy: 0.0952 | Regression_eta: 0.0556 | Regression_pt: 0.0999 | Regression_sin_phi: 0.1050 | LR: 8.88e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3394s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank4 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank4 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1106 | Classification_binary: 0.2951 | Regression_cos_phi: 0.1275 | Regression_energy: 0.1022 | Regression_eta: 0.1023 | Regression_pt: 0.1070 | Regression_sin_phi: 0.1350 | LR: 8.86e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank4 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank4 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0928 | Regression_eta: 0.0522 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0854 | LR: 8.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank4 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank4 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0765 | Regression_energy: 0.1009 | Regression_eta: 0.0528 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0852 | LR: 8.82e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5599s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank4 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank4 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1047 | Classification_binary: 0.2326 | Regression_cos_phi: 0.1211 | Regression_energy: 0.0932 | Regression_eta: 0.0874 | Regression_pt: 0.0974 | Regression_sin_phi: 0.1433 | LR: 8.80e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank4 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank4 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2891 | Regression_cos_phi: 0.1172 | Regression_energy: 0.1166 | Regression_eta: 0.0735 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0958 | LR: 8.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0207s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank4 [2026-10-28 21:36:38,362] INFO : Running validation on rank4 +rank4 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:37:31,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 21:38:00,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:38:01,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:38:02,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:38:03,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:38:13,674] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:38:14,611] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 21:38:30,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:38:33,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:38:34,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:38:35,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:38:45,250] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 21:38:45,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:38:46,116] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 21:39:02,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:39:03,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:39:03,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:39:04,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:39:14,722] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank4 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2372 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0955 | Regression_eta: 0.1175 | Regression_pt: 0.0971 | Regression_sin_phi: 0.1315 | LR: 8.76e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1500s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank4 [2026-10-28 21:49:00,968] INFO : Step 12200/50000 rank4 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.1861 | Regression_cos_phi: 0.1139 | Regression_energy: 0.0857 | Regression_eta: 0.0953 | Regression_pt: 0.0903 | Regression_sin_phi: 0.1063 | LR: 8.74e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.5157s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank4 [2026-10-28 21:53:55,879] INFO : Step 12300/50000 rank4 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.2622 | Regression_cos_phi: 0.1283 | Regression_energy: 0.0873 | Regression_eta: 0.0861 | Regression_pt: 0.0902 | Regression_sin_phi: 0.1212 | LR: 8.72e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6877s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank4 [2026-10-28 21:58:40,958] INFO : Step 12400/50000 rank4 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2752 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1005 | Regression_eta: 0.0658 | Regression_pt: 0.1023 | Regression_sin_phi: 0.1183 | LR: 8.70e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.2687s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank4 [2026-10-28 22:03:23,386] INFO : Step 12500/50000 rank4 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0987 | Regression_eta: 0.0616 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0851 | LR: 8.68e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6728s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank4 [2026-10-28 22:08:04,812] INFO : Step 12600/50000 rank4 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2376 | Regression_cos_phi: 0.1207 | Regression_energy: 0.1033 | Regression_eta: 0.0765 | Regression_pt: 0.1073 | Regression_sin_phi: 0.1328 | LR: 8.66e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1055s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank4 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank4 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2744 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0883 | Regression_eta: 0.0477 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0882 | LR: 8.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3608s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank4 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank4 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2169 | Regression_cos_phi: 0.1236 | Regression_energy: 0.1156 | Regression_eta: 0.0815 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0902 | LR: 8.62e-04 | DataLoad Time: 0.0234s | Model Forward Time: 6.0963s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank4 [2026-10-28 22:22:32,703] INFO : Step 12900/50000 rank4 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2296 | Regression_cos_phi: 0.1206 | Regression_energy: 0.0983 | Regression_eta: 0.0731 | Regression_pt: 0.1013 | Regression_sin_phi: 0.1144 | LR: 8.60e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6628s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank4 [2026-10-28 22:27:21,815] INFO : Step 13000/50000 rank4 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0804 | Regression_eta: 0.0692 | Regression_pt: 0.0839 | Regression_sin_phi: 0.1047 | LR: 8.58e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.9248s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank4 [2026-10-28 22:27:23,210] INFO : Running validation on rank4 +rank4 [2026-10-28 22:27:23,210] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 22:27:23,210] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:28:28,174] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 22:28:54,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 22:28:55,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 22:28:56,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 22:28:57,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 22:29:07,317] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 22:29:08,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:29:08,604] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 22:29:24,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 22:29:25,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 22:29:26,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 22:29:27,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 22:29:37,665] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:29:38,495] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 22:29:54,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 22:29:55,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 22:29:56,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 22:29:57,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 22:30:07,090] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 22:35:15,708] INFO : Step 13100/50000 rank4 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2373 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0970 | Regression_eta: 0.0723 | Regression_pt: 0.0998 | Regression_sin_phi: 0.1227 | LR: 8.56e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.4860s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank4 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank4 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0980 | Regression_eta: 0.0599 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0669 | LR: 8.54e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6864s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank4 [2026-10-28 22:44:58,539] INFO : Step 13300/50000 rank4 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0795 | Regression_eta: 0.0493 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0922 | LR: 8.52e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6757s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank4 [2026-10-28 22:49:50,192] INFO : Step 13400/50000 rank4 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.3179 | Regression_cos_phi: 0.1049 | Regression_energy: 0.1251 | Regression_eta: 0.0891 | Regression_pt: 0.1287 | Regression_sin_phi: 0.1621 | LR: 8.50e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.6043s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank4 [2026-10-28 22:54:39,510] INFO : Step 13500/50000 rank4 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0845 | Regression_eta: 0.0569 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0781 | LR: 8.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4628s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank4 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank4 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0814 | Regression_eta: 0.0490 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0654 | LR: 8.45e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8504s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank4 [2026-10-28 23:04:05,987] INFO : Step 13700/50000 rank4 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.2519 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0965 | Regression_eta: 0.0504 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0805 | LR: 8.43e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7324s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank4 [2026-10-28 23:09:16,293] INFO : Step 13800/50000 rank4 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2261 | Regression_cos_phi: 0.1115 | Regression_energy: 0.0872 | Regression_eta: 0.0548 | Regression_pt: 0.0890 | Regression_sin_phi: 0.1112 | LR: 8.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4793s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank4 [2026-10-28 23:14:10,154] INFO : Step 13900/50000 rank4 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0971 | Classification_binary: 0.2160 | Regression_cos_phi: 0.1040 | Regression_energy: 0.1174 | Regression_eta: 0.0563 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0810 | LR: 8.39e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2899s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank4 [2026-10-28 23:18:59,722] INFO : Step 14000/50000 rank4 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2494 | Regression_cos_phi: 0.1053 | Regression_energy: 0.0902 | Regression_eta: 0.0688 | Regression_pt: 0.0936 | Regression_sin_phi: 0.1100 | LR: 8.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1756s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank4 [2026-10-28 23:19:00,860] INFO : Running validation on rank4 +rank4 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:19:53,516] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-28 23:20:19,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 23:20:20,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 23:20:21,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 23:20:22,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 23:20:32,590] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:20:33,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:20:33,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-28 23:20:49,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 23:20:50,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 23:20:51,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 23:20:52,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 23:21:02,574] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:21:03,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:21:04,038] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-28 23:21:19,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 23:21:20,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 23:21:21,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 23:21:22,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 23:21:32,797] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:26:43,450] INFO : Step 14100/50000 rank4 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2208 | Regression_cos_phi: 0.1161 | Regression_energy: 0.0887 | Regression_eta: 0.0828 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0873 | LR: 8.35e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.5294s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank4 [2026-10-28 23:31:31,720] INFO : Step 14200/50000 rank4 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0852 | Regression_eta: 0.0445 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0625 | LR: 8.32e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6165s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank4 [2026-10-28 23:36:16,487] INFO : Step 14300/50000 rank4 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0838 | Regression_eta: 0.0493 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0981 | LR: 8.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6736s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank4 [2026-10-28 23:40:58,337] INFO : Step 14400/50000 rank4 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.2403 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0885 | Regression_eta: 0.0512 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0917 | LR: 8.28e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7758s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank4 [2026-10-28 23:45:42,941] INFO : Step 14500/50000 rank4 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0808 | Regression_eta: 0.0461 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0469 | LR: 8.26e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6362s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank4 [2026-10-28 23:50:27,494] INFO : Step 14600/50000 rank4 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0870 | Regression_eta: 0.0573 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0610 | LR: 8.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4485s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank4 [2026-10-28 23:55:12,844] INFO : Step 14700/50000 rank4 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2812 | Regression_cos_phi: 0.1054 | Regression_energy: 0.0888 | Regression_eta: 0.0703 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0922 | LR: 8.21e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6470s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank4 [2026-10-28 23:59:56,486] INFO : Step 14800/50000 rank4 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0735 | Regression_eta: 0.0571 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0548 | LR: 8.19e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.7617s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank4 [2026-10-29 00:04:46,438] INFO : Step 14900/50000 rank4 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0945 | Regression_eta: 0.0608 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0976 | LR: 8.17e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.1954s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank4 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank4 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2718 | Regression_cos_phi: 0.1014 | Regression_energy: 0.0909 | Regression_eta: 0.0780 | Regression_pt: 0.0947 | Regression_sin_phi: 0.1060 | LR: 8.15e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7490s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank4 [2026-10-29 00:09:44,171] INFO : Running validation on rank4 +rank4 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:10:38,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:10:39,316] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 00:11:05,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:11:07,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:11:08,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:11:09,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:11:19,466] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 00:11:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:11:20,774] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 00:11:36,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:11:37,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:11:38,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:11:39,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:11:49,873] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:11:50,986] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 00:12:06,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:12:07,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:12:08,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:12:09,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:12:19,322] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:17:30,255] INFO : Step 15100/50000 rank4 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2413 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0883 | Regression_eta: 0.0589 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0805 | LR: 8.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3992s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank4 [2026-10-29 00:22:16,917] INFO : Step 15200/50000 rank4 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1671 | Regression_cos_phi: 0.1099 | Regression_energy: 0.1116 | Regression_eta: 0.0768 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0885 | LR: 8.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2239s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank4 [2026-10-29 00:27:05,719] INFO : Step 15300/50000 rank4 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0904 | Regression_eta: 0.0461 | Regression_pt: 0.0969 | Regression_sin_phi: 0.1017 | LR: 8.08e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank4 [2026-10-29 00:31:57,726] INFO : Step 15400/50000 rank4 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1136 | Regression_energy: 0.0871 | Regression_eta: 0.0538 | Regression_pt: 0.0940 | Regression_sin_phi: 0.1194 | LR: 8.05e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6643s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank4 [2026-10-29 00:36:52,791] INFO : Step 15500/50000 rank4 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0850 | Regression_eta: 0.0573 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0694 | LR: 8.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4232s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank4 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank4 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0729 | Regression_eta: 0.0672 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0909 | LR: 8.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9288s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank4 [2026-10-29 00:46:43,873] INFO : Step 15700/50000 rank4 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2079 | Regression_cos_phi: 0.1154 | Regression_energy: 0.1005 | Regression_eta: 0.0551 | Regression_pt: 0.1034 | Regression_sin_phi: 0.1076 | LR: 7.98e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.7554s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank4 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank4 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2254 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0872 | Regression_eta: 0.0638 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0746 | LR: 7.96e-04 | DataLoad Time: 0.0241s | Model Forward Time: 5.0238s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank4 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank4 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1719 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0723 | Regression_eta: 0.0573 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0823 | LR: 7.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank4 [2026-10-29 01:01:16,824] INFO : Step 16000/50000 rank4 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2367 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0754 | Regression_eta: 0.0682 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0742 | LR: 7.91e-04 | DataLoad Time: 0.0228s | Model Forward Time: 5.3002s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank4 [2026-10-29 01:01:17,883] INFO : Running validation on rank4 +rank4 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 01:02:10,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:02:11,391] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 01:02:39,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 01:02:41,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 01:02:43,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 01:02:45,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 01:02:55,017] INFO : Time taken to make predictions on device 4 is: 0.29 min +rank4 [2026-10-29 01:02:55,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:02:55,636] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 01:03:11,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 01:03:12,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 01:03:13,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 01:03:15,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 01:03:25,066] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 01:03:25,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:03:26,084] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 01:03:41,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 01:03:42,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 01:03:43,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 01:03:44,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 01:03:54,194] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank4 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2501 | Regression_cos_phi: 0.1189 | Regression_energy: 0.0766 | Regression_eta: 0.0729 | Regression_pt: 0.0793 | Regression_sin_phi: 0.1007 | LR: 7.89e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0657s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank4 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank4 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0733 | Regression_eta: 0.0460 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0559 | LR: 7.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2115s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank4 [2026-10-29 01:19:03,892] INFO : Step 16300/50000 rank4 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0715 | Regression_eta: 0.0612 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0756 | LR: 7.84e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.2097s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank4 [2026-10-29 01:23:56,184] INFO : Step 16400/50000 rank4 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0740 | Regression_eta: 0.0462 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0765 | LR: 7.81e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6851s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank4 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank4 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0908 | Regression_eta: 0.0509 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0689 | LR: 7.79e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3068s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank4 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank4 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0814 | Regression_eta: 0.0463 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0713 | LR: 7.77e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1714s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank4 [2026-10-29 01:38:38,620] INFO : Step 16700/50000 rank4 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.1969 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0793 | Regression_eta: 0.0445 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0842 | LR: 7.74e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.9122s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank4 [2026-10-29 01:43:30,910] INFO : Step 16800/50000 rank4 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0863 | Regression_eta: 0.0533 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0931 | LR: 7.72e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.5995s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank4 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank4 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0679 | Regression_eta: 0.0549 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0808 | LR: 7.69e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3823s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank4 [2026-10-29 01:53:19,525] INFO : Step 17000/50000 rank4 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0918 | Regression_eta: 0.0697 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0912 | LR: 7.67e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.6711s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank4 [2026-10-29 01:53:20,625] INFO : Running validation on rank4 +rank4 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 01:54:26,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:54:27,391] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 01:55:04,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 01:55:05,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 01:55:07,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 01:55:08,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 01:55:18,024] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:55:19,228] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 01:55:35,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 01:55:36,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 01:55:38,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 01:55:39,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 01:55:49,878] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:55:50,414] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 01:56:06,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 01:56:06,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 01:56:07,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 01:56:08,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 01:56:18,553] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:01:32,243] INFO : Step 17100/50000 rank4 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0818 | Regression_eta: 0.0411 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0503 | LR: 7.64e-04 | DataLoad Time: 0.0150s | Model Forward Time: 6.1172s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank4 [2026-10-29 02:06:24,502] INFO : Step 17200/50000 rank4 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0660 | Regression_eta: 0.0524 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0640 | LR: 7.62e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.6223s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank4 [2026-10-29 02:11:06,510] INFO : Step 17300/50000 rank4 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0814 | Regression_eta: 0.0544 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0858 | LR: 7.59e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7532s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank4 [2026-10-29 02:16:07,514] INFO : Step 17400/50000 rank4 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2316 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0799 | Regression_eta: 0.0576 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0815 | LR: 7.57e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5466s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank4 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank4 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0934 | Regression_eta: 0.0614 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0873 | LR: 7.54e-04 | DataLoad Time: 0.0139s | Model Forward Time: 5.2354s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank4 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank4 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0697 | Regression_eta: 0.0387 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0648 | LR: 7.52e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.4703s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank4 [2026-10-29 02:30:47,052] INFO : Step 17700/50000 rank4 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0784 | Regression_eta: 0.0524 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0840 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7701s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank4 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank4 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0780 | Regression_eta: 0.0448 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0789 | LR: 7.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0205s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank4 [2026-10-29 02:40:30,747] INFO : Step 17900/50000 rank4 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0757 | Regression_eta: 0.0369 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0548 | LR: 7.44e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank4 [2026-10-29 02:45:16,091] INFO : Step 18000/50000 rank4 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0775 | Regression_eta: 0.0430 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0700 | LR: 7.42e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6592s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank4 [2026-10-29 02:45:17,190] INFO : Running validation on rank4 +rank4 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:46:12,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:46:13,323] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 02:46:39,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:46:40,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:46:41,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:46:42,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:46:52,641] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:46:53,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 02:47:09,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:47:10,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:47:12,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:47:14,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:47:24,490] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 02:47:25,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:47:25,821] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 02:47:41,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:47:42,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:47:43,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:47:44,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:47:54,232] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:53:14,469] INFO : Step 18100/50000 rank4 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0633 | Regression_eta: 0.0552 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0810 | LR: 7.39e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7386s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank4 [2026-10-29 02:58:08,645] INFO : Step 18200/50000 rank4 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0726 | Regression_energy: 0.1102 | Regression_eta: 0.0802 | Regression_pt: 0.1135 | Regression_sin_phi: 0.1003 | LR: 7.36e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9653s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank4 [2026-10-29 03:02:55,066] INFO : Step 18300/50000 rank4 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0820 | Regression_eta: 0.0523 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0761 | LR: 7.34e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5938s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank4 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank4 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2254 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0734 | Regression_eta: 0.0510 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0654 | LR: 7.31e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7130s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank4 [2026-10-29 03:12:43,564] INFO : Step 18500/50000 rank4 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0780 | Regression_eta: 0.0520 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0751 | LR: 7.29e-04 | DataLoad Time: 0.0138s | Model Forward Time: 5.3600s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank4 [2026-10-29 03:17:39,474] INFO : Step 18600/50000 rank4 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0819 | Regression_eta: 0.1105 | Regression_pt: 0.0827 | Regression_sin_phi: 0.2424 | LR: 7.26e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.4732s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank4 [2026-10-29 03:22:31,158] INFO : Step 18700/50000 rank4 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0771 | Regression_eta: 0.0614 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0693 | LR: 7.24e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4817s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank4 [2026-10-29 03:27:37,245] INFO : Step 18800/50000 rank4 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0779 | Regression_eta: 0.0343 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0720 | LR: 7.21e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.9531s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank4 [2026-10-29 03:32:26,944] INFO : Step 18900/50000 rank4 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0594 | Regression_eta: 0.0779 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0771 | LR: 7.18e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5370s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank4 [2026-10-29 03:37:06,569] INFO : Step 19000/50000 rank4 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0824 | Regression_eta: 0.0516 | Regression_pt: 0.0856 | Regression_sin_phi: 0.1129 | LR: 7.16e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5304s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank4 [2026-10-29 03:37:07,713] INFO : Running validation on rank4 +rank4 [2026-10-29 03:37:07,713] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 03:38:00,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:38:00,820] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 03:38:27,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 03:38:28,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 03:38:29,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 03:38:30,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 03:38:40,498] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 03:38:40,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:38:41,425] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 03:38:57,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 03:38:58,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 03:38:59,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 03:39:00,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 03:39:10,494] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:39:11,226] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 03:39:26,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 03:39:27,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 03:39:28,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 03:39:29,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 03:39:39,081] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank4 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0728 | Regression_eta: 0.0402 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0475 | LR: 7.13e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.9870s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank4 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank4 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0768 | Regression_eta: 0.0513 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0513 | LR: 7.10e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5597s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank4 [2026-10-29 03:54:09,159] INFO : Step 19300/50000 rank4 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0619 | Regression_eta: 0.0472 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0839 | LR: 7.08e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3959s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank4 [2026-10-29 03:58:51,217] INFO : Step 19400/50000 rank4 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2201 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0669 | Regression_eta: 0.0543 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0700 | LR: 7.05e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6161s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank4 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank4 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.2007 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0724 | Regression_eta: 0.0489 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0661 | LR: 7.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7169s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank4 [2026-10-29 04:08:12,466] INFO : Step 19600/50000 rank4 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0836 | Regression_eta: 0.0395 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0736 | LR: 7.00e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4200s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank4 [2026-10-29 04:12:52,348] INFO : Step 19700/50000 rank4 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0665 | Regression_eta: 0.0515 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0856 | LR: 6.97e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3576s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank4 [2026-10-29 04:17:28,315] INFO : Step 19800/50000 rank4 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0550 | Regression_eta: 0.0454 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0483 | LR: 6.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0460s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank4 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank4 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1833 | Regression_cos_phi: 0.1081 | Regression_energy: 0.0787 | Regression_eta: 0.0857 | Regression_pt: 0.0836 | Regression_sin_phi: 0.1384 | LR: 6.92e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.3169s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank4 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank4 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0666 | Regression_eta: 0.0420 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0690 | LR: 6.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6555s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank4 [2026-10-29 04:26:52,970] INFO : Running validation on rank4 +rank4 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:27:46,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:27:46,652] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 04:28:14,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:28:15,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:28:16,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:28:17,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:28:27,406] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:28:28,501] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 04:28:44,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:28:45,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:28:46,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:28:48,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:28:58,016] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:28:59,247] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 04:29:14,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:29:15,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:29:16,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:29:17,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:29:27,436] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 04:34:27,055] INFO : Step 20100/50000 rank4 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0662 | Regression_eta: 0.0377 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0706 | LR: 6.86e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.3248s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank4 [2026-10-29 04:39:08,148] INFO : Step 20200/50000 rank4 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0652 | Regression_eta: 0.0506 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0942 | LR: 6.84e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6180s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank4 [2026-10-29 04:43:47,102] INFO : Step 20300/50000 rank4 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0568 | Regression_eta: 0.0426 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0961 | LR: 6.81e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4950s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank4 [2026-10-29 04:48:26,732] INFO : Step 20400/50000 rank4 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0666 | Regression_eta: 0.0506 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0587 | LR: 6.78e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4417s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank4 [2026-10-29 04:53:08,466] INFO : Step 20500/50000 rank4 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2250 | Regression_cos_phi: 0.1104 | Regression_energy: 0.0779 | Regression_eta: 0.0758 | Regression_pt: 0.0814 | Regression_sin_phi: 0.1108 | LR: 6.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0623s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank4 [2026-10-29 04:57:53,290] INFO : Step 20600/50000 rank4 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0589 | Regression_eta: 0.0446 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0682 | LR: 6.73e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7528s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank4 [2026-10-29 05:02:47,890] INFO : Step 20700/50000 rank4 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0695 | Regression_eta: 0.0475 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0775 | LR: 6.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6380s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank4 [2026-10-29 05:07:34,620] INFO : Step 20800/50000 rank4 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2193 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0752 | Regression_eta: 0.0388 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0858 | LR: 6.67e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.8153s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank4 [2026-10-29 05:12:18,487] INFO : Step 20900/50000 rank4 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0501 | Regression_eta: 0.0410 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0609 | LR: 6.65e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6299s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank4 [2026-10-29 05:17:03,998] INFO : Step 21000/50000 rank4 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0763 | Regression_eta: 0.0532 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0714 | LR: 6.62e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank4 [2026-10-29 05:17:04,884] INFO : Running validation on rank4 +rank4 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 05:17:58,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:17:58,805] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 05:18:22,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 05:18:24,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 05:18:25,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 05:18:26,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 05:18:36,142] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:18:37,118] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 05:18:53,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 05:18:54,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 05:18:55,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 05:18:56,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 05:19:06,067] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 05:19:06,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:19:07,278] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 05:19:23,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 05:19:24,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 05:19:24,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 05:19:25,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 05:19:35,745] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank4 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0669 | Regression_eta: 0.0364 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0499 | LR: 6.59e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.3802s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank4 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank4 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 05:29:24,523] INFO : Step 21200/50000 rank4 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1334 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0715 | Regression_eta: 0.0325 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0449 | LR: 6.56e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6428s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank4 [2026-10-29 05:34:39,116] INFO : Step 21300/50000 rank4 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0541 | Regression_eta: 0.0458 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0530 | LR: 6.54e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6744s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank4 [2026-10-29 05:39:29,689] INFO : Step 21400/50000 rank4 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0752 | Regression_eta: 0.0559 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0842 | LR: 6.51e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3961s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank4 [2026-10-29 05:44:17,736] INFO : Step 21500/50000 rank4 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0222 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0763 | Regression_eta: 0.0360 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0447 | LR: 6.48e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3084s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank4 [2026-10-29 05:49:11,032] INFO : Step 21600/50000 rank4 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2235 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0804 | Regression_eta: 0.0580 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0924 | LR: 6.45e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5078s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank4 [2026-10-29 05:53:59,566] INFO : Step 21700/50000 rank4 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0713 | Regression_eta: 0.0355 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0671 | LR: 6.43e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5564s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank4 [2026-10-29 05:58:52,461] INFO : Step 21800/50000 rank4 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0792 | Regression_energy: 0.0613 | Regression_eta: 0.0634 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0494 | LR: 6.40e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1518s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank4 [2026-10-29 06:03:49,408] INFO : Step 21900/50000 rank4 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0742 | Regression_eta: 0.0396 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0511 | LR: 6.37e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8535s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank4 [2026-10-29 06:08:50,709] INFO : Step 22000/50000 rank4 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2421 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0622 | Regression_eta: 0.0578 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0897 | LR: 6.34e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4728s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank4 [2026-10-29 06:08:51,802] INFO : Running validation on rank4 +rank4 [2026-10-29 06:08:51,802] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 06:08:51,803] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 06:09:45,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:09:46,124] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 06:10:12,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 06:10:13,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 06:10:14,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 06:10:15,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 06:10:25,891] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 06:10:26,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 06:10:43,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 06:10:43,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 06:10:44,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 06:10:45,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 06:10:55,912] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 06:10:56,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:10:57,441] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 06:11:13,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 06:11:14,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 06:11:15,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 06:11:15,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 06:11:25,930] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank4 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0760 | Regression_eta: 0.0380 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0538 | LR: 6.32e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.1902s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank4 [2026-10-29 06:21:31,723] INFO : Step 22200/50000 rank4 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0553 | Regression_eta: 0.0284 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0343 | LR: 6.29e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.2922s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank4 [2026-10-29 06:26:33,414] INFO : Step 22300/50000 rank4 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0621 | Regression_eta: 0.0390 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0651 | LR: 6.26e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7521s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank4 [2026-10-29 06:31:29,807] INFO : Step 22400/50000 rank4 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0589 | Regression_eta: 0.0450 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0658 | LR: 6.23e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.4001s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank4 [2026-10-29 06:36:31,515] INFO : Step 22500/50000 rank4 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0658 | Regression_eta: 0.0299 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0600 | LR: 6.20e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2545s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank4 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank4 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0623 | Regression_eta: 0.0387 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0683 | LR: 6.18e-04 | DataLoad Time: 0.0256s | Model Forward Time: 4.3117s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank4 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank4 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0694 | Regression_eta: 0.0528 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0597 | LR: 6.15e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9443s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank4 [2026-10-29 06:51:26,821] INFO : Step 22800/50000 rank4 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1949 | Regression_cos_phi: 0.1222 | Regression_energy: 0.0685 | Regression_eta: 0.0824 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0813 | LR: 6.12e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.4052s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank4 [2026-10-29 06:56:23,354] INFO : Step 22900/50000 rank4 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0588 | Regression_eta: 0.0370 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0546 | LR: 6.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8118s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank4 [2026-10-29 07:01:33,063] INFO : Step 23000/50000 rank4 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0663 | Regression_eta: 0.0337 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0665 | LR: 6.06e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4757s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank4 [2026-10-29 07:01:34,091] INFO : Running validation on rank4 +rank4 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:02:28,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 07:02:49,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:02:50,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:02:51,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:02:52,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:03:02,653] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:03:03,431] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:03:03,707] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 07:03:19,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:03:20,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:03:21,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:03:22,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:03:32,982] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 07:03:33,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:03:34,171] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 07:03:49,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:03:50,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:03:51,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:03:52,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:04:02,440] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 07:09:13,971] INFO : Step 23100/50000 rank4 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0696 | Regression_eta: 0.0399 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0576 | LR: 6.04e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.1796s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank4 [2026-10-29 07:14:12,718] INFO : Step 23200/50000 rank4 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1250 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0652 | Regression_eta: 0.0404 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0504 | LR: 6.01e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.6115s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank4 [2026-10-29 07:19:05,798] INFO : Step 23300/50000 rank4 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0482 | Regression_eta: 0.0401 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0573 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8591s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank4 [2026-10-29 07:24:01,132] INFO : Step 23400/50000 rank4 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0692 | Regression_eta: 0.0419 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0979 | LR: 5.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0099s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank4 [2026-10-29 07:29:14,711] INFO : Step 23500/50000 rank4 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0679 | Regression_eta: 0.0421 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0564 | LR: 5.92e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8342s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank4 [2026-10-29 07:34:22,319] INFO : Step 23600/50000 rank4 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0614 | Regression_eta: 0.0466 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0512 | LR: 5.90e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.3912s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank4 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank4 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0700 | Regression_eta: 0.0468 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0819 | LR: 5.87e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.8162s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank4 [2026-10-29 07:44:46,684] INFO : Step 23800/50000 rank4 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0636 | Regression_eta: 0.0546 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0643 | LR: 5.84e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8789s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank4 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank4 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0682 | Regression_eta: 0.0404 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0615 | LR: 5.81e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.8163s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank4 [2026-10-29 07:54:58,899] INFO : Step 24000/50000 rank4 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2104 | Regression_cos_phi: 0.1075 | Regression_energy: 0.0649 | Regression_eta: 0.0480 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0682 | LR: 5.78e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8568s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank4 [2026-10-29 07:54:59,832] INFO : Running validation on rank4 +rank4 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:55:54,113] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 07:56:20,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:56:21,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:56:23,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:56:24,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:56:34,086] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:56:35,055] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 07:56:51,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:56:52,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:56:53,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:56:54,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:57:04,125] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:57:05,693] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 07:57:21,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:57:22,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:57:22,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:57:23,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:57:33,731] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 08:03:04,662] INFO : Step 24100/50000 rank4 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2218 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0626 | Regression_eta: 0.0518 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0722 | LR: 5.75e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5904s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank4 [2026-10-29 08:07:59,796] INFO : Step 24200/50000 rank4 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0608 | Regression_eta: 0.0382 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0676 | LR: 5.73e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0934s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank4 [2026-10-29 08:13:18,507] INFO : Step 24300/50000 rank4 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0669 | Regression_eta: 0.0359 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0701 | LR: 5.70e-04 | DataLoad Time: 0.0218s | Model Forward Time: 7.4895s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank4 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank4 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2164 | Regression_cos_phi: 0.1236 | Regression_energy: 0.0624 | Regression_eta: 0.0858 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0809 | LR: 5.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1593s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank4 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank4 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0587 | Regression_eta: 0.0524 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0502 | LR: 5.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5200s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank4 [2026-10-29 08:28:18,515] INFO : Step 24600/50000 rank4 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0718 | Regression_eta: 0.0426 | Regression_pt: 0.0754 | Regression_sin_phi: 0.1045 | LR: 5.61e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7042s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank4 [2026-10-29 08:33:21,874] INFO : Step 24700/50000 rank4 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0590 | Regression_eta: 0.0667 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0940 | LR: 5.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9282s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank4 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank4 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0875 | Regression_eta: 0.0389 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0518 | LR: 5.56e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.2069s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank4 [2026-10-29 08:43:21,524] INFO : Step 24900/50000 rank4 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0800 | Regression_eta: 0.0354 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0569 | LR: 5.53e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4730s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank4 [2026-10-29 08:48:25,835] INFO : Step 25000/50000 rank4 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0644 | Regression_eta: 0.0367 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0661 | LR: 5.50e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9539s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank4 [2026-10-29 08:48:26,771] INFO : Running validation on rank4 +rank4 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 08:49:20,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:49:21,192] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 08:49:51,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 08:49:53,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 08:49:54,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 08:49:55,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 08:50:05,767] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:50:06,730] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 08:50:22,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 08:50:23,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 08:50:24,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 08:50:25,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 08:50:35,874] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:50:36,987] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 08:50:53,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 08:50:53,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 08:50:54,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 08:50:55,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 08:51:05,847] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 08:56:18,538] INFO : Step 25100/50000 rank4 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0470 | Regression_eta: 0.0537 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0721 | LR: 5.47e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.3800s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank4 [2026-10-29 09:01:01,591] INFO : Step 25200/50000 rank4 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1318 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0575 | Regression_eta: 0.0336 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0575 | LR: 5.44e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8758s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank4 [2026-10-29 09:05:50,664] INFO : Step 25300/50000 rank4 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0554 | Regression_eta: 0.0433 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0601 | LR: 5.42e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.9767s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank4 [2026-10-29 09:10:30,565] INFO : Step 25400/50000 rank4 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0739 | Regression_eta: 0.0596 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0849 | LR: 5.39e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.1691s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank4 [2026-10-29 09:15:11,238] INFO : Step 25500/50000 rank4 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1897 | Regression_cos_phi: 0.1055 | Regression_energy: 0.0826 | Regression_eta: 0.0542 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0795 | LR: 5.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4701s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank4 [2026-10-29 09:19:56,597] INFO : Step 25600/50000 rank4 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0739 | Regression_eta: 0.0558 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0849 | LR: 5.33e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.9351s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank4 [2026-10-29 09:24:33,865] INFO : Step 25700/50000 rank4 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0780 | Regression_eta: 0.0525 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0857 | LR: 5.30e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2138s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank4 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank4 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0546 | Regression_eta: 0.0361 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0593 | LR: 5.27e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1665s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank4 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank4 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0560 | Regression_eta: 0.0457 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0656 | LR: 5.25e-04 | DataLoad Time: 0.0250s | Model Forward Time: 4.7706s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank4 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank4 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0971 | Regression_eta: 0.0442 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0825 | LR: 5.22e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7499s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank4 [2026-10-29 09:39:04,383] INFO : Running validation on rank4 +rank4 [2026-10-29 09:39:04,383] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:39:04,383] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:39:55,910] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 09:40:23,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:40:24,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:40:25,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:40:26,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:40:36,497] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:40:37,505] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 09:40:54,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:40:56,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:40:57,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:40:58,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:41:08,070] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 09:41:08,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:44:09,582] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 09:44:24,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:44:25,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:44:26,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:44:27,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:44:37,721] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 09:50:29,555] INFO : Step 26100/50000 rank4 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2226 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0610 | Regression_eta: 0.0541 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0893 | LR: 5.19e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1797s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank4 [2026-10-29 09:55:15,273] INFO : Step 26200/50000 rank4 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0631 | Regression_eta: 0.0329 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0520 | LR: 5.16e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8128s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank4 [2026-10-29 09:59:57,744] INFO : Step 26300/50000 rank4 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0438 | Regression_eta: 0.0423 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0624 | LR: 5.13e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.9304s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank4 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank4 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0567 | Regression_eta: 0.0448 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0728 | LR: 5.10e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.1498s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank4 [2026-10-29 10:09:23,739] INFO : Step 26500/50000 rank4 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0552 | Regression_eta: 0.0580 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0750 | LR: 5.08e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0941s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank4 [2026-10-29 10:14:05,781] INFO : Step 26600/50000 rank4 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1350 | Regression_cos_phi: 0.1136 | Regression_energy: 0.0615 | Regression_eta: 0.0438 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0670 | LR: 5.05e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6623s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank4 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank4 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0973 | Regression_energy: 0.0682 | Regression_eta: 0.0526 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0844 | LR: 5.02e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.2161s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank4 [2026-10-29 10:23:58,675] INFO : Step 26800/50000 rank4 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0617 | Regression_eta: 0.0368 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0717 | LR: 4.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5308s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank4 [2026-10-29 10:28:51,387] INFO : Step 26900/50000 rank4 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0604 | Regression_eta: 0.0402 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0671 | LR: 4.96e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5632s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank4 [2026-10-29 10:33:37,219] INFO : Step 27000/50000 rank4 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0765 | Regression_eta: 0.0479 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0928 | LR: 4.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6920s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank4 [2026-10-29 10:33:38,215] INFO : Running validation on rank4 +rank4 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 10:34:33,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:34:34,220] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 10:35:00,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 10:35:01,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 10:35:02,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 10:35:03,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 10:35:13,342] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:35:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:35:14,483] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 10:35:30,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 10:35:31,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 10:35:32,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 10:35:33,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 10:35:43,591] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:35:44,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:35:45,038] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 10:36:00,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 10:36:01,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 10:36:02,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 10:36:03,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 10:36:13,205] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 10:41:28,285] INFO : Step 27100/50000 rank4 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0501 | Regression_eta: 0.0560 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0693 | LR: 4.91e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6737s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank4 [2026-10-29 10:46:22,294] INFO : Step 27200/50000 rank4 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0634 | Regression_eta: 0.0509 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0675 | LR: 4.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8267s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank4 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank4 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0559 | Regression_eta: 0.0349 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0738 | LR: 4.85e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6707s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank4 [2026-10-29 10:56:31,406] INFO : Step 27400/50000 rank4 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0663 | Regression_eta: 0.0360 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0726 | LR: 4.82e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9759s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank4 [2026-10-29 11:01:35,254] INFO : Step 27500/50000 rank4 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0609 | Regression_eta: 0.0390 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0646 | LR: 4.80e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.3166s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank4 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank4 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0477 | Regression_eta: 0.0422 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0459 | LR: 4.77e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.3320s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank4 [2026-10-29 11:13:33,056] INFO : Step 27700/50000 rank4 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0708 | Regression_eta: 0.0278 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0450 | LR: 4.74e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7948s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank4 [2026-10-29 11:18:30,827] INFO : Step 27800/50000 rank4 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0693 | Regression_eta: 0.0477 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0647 | LR: 4.71e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0128s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank4 [2026-10-29 11:23:38,363] INFO : Step 27900/50000 rank4 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0667 | Regression_eta: 0.0352 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0391 | LR: 4.68e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8877s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank4 [2026-10-29 11:28:54,872] INFO : Step 28000/50000 rank4 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0565 | Regression_eta: 0.0452 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0657 | LR: 4.66e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9111s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank4 [2026-10-29 11:28:55,940] INFO : Running validation on rank4 +rank4 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 11:29:47,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:29:48,525] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 11:30:14,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 11:30:15,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 11:30:16,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 11:30:17,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 11:30:27,778] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:30:28,727] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 11:30:44,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 11:30:45,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 11:30:46,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 11:30:47,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 11:30:57,674] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 11:30:57,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:30:58,465] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 11:31:13,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 11:31:14,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 11:31:15,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 11:31:16,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 11:31:26,471] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 11:37:01,728] INFO : Step 28100/50000 rank4 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0577 | Regression_eta: 0.0455 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0659 | LR: 4.63e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.5021s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank4 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank4 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0558 | Regression_eta: 0.0546 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0520 | LR: 4.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.0739s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank4 [2026-10-29 11:47:09,276] INFO : Step 28300/50000 rank4 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0695 | Regression_eta: 0.0476 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0459 | LR: 4.57e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.0936s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank4 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank4 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1478 | Classification_binary: 0.2172 | Regression_cos_phi: 0.0819 | Regression_energy: 0.1220 | Regression_eta: 0.0697 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0727 | LR: 4.55e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5894s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank4 [2026-10-29 11:56:56,841] INFO : Step 28500/50000 rank4 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0554 | Regression_eta: 0.0587 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0478 | LR: 4.52e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.9872s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank4 [2026-10-29 12:01:46,922] INFO : Step 28600/50000 rank4 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0210 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0716 | Regression_eta: 0.0333 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0459 | LR: 4.49e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6580s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank4 [2026-10-29 12:06:39,995] INFO : Step 28700/50000 rank4 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0516 | Regression_eta: 0.0398 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0505 | LR: 4.46e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.5128s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank4 [2026-10-29 12:11:31,023] INFO : Step 28800/50000 rank4 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0646 | Regression_eta: 0.0492 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0468 | LR: 4.44e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.9311s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank4 [2026-10-29 12:16:17,097] INFO : Step 28900/50000 rank4 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0488 | Regression_eta: 0.0377 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0639 | LR: 4.41e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7006s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank4 [2026-10-29 12:21:10,133] INFO : Step 29000/50000 rank4 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0598 | Regression_eta: 0.0410 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0561 | LR: 4.38e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9915s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank4 [2026-10-29 12:21:11,231] INFO : Running validation on rank4 +rank4 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 12:22:31,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 12:22:32,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 12:22:33,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 12:22:35,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 12:22:45,199] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:22:45,974] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 12:23:01,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 12:23:02,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 12:23:03,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 12:23:04,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 12:23:14,830] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 12:23:15,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:23:16,444] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 12:23:32,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 12:23:32,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 12:23:33,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 12:23:34,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 12:23:44,586] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank4 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0560 | Regression_eta: 0.0403 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0893 | LR: 4.35e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.1733s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank4 [2026-10-29 12:33:43,194] INFO : Step 29200/50000 rank4 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0478 | Regression_eta: 0.0457 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0457 | LR: 4.33e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5659s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank4 [2026-10-29 14:49:26,361] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-29 14:49:26,370] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank4 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-29 14:49:54,088] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank4 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank4 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank4 [2026-10-29 14:49:54,133] INFO : Initializing backbone layer 2 +rank4 [2026-10-29 14:49:54,141] INFO : Initializing backbone layer 3 +rank4 [2026-10-29 14:49:54,150] INFO : Initializing backbone layer 4 +rank4 [2026-10-29 14:49:54,158] INFO : Initializing backbone layer 5 +rank4 [2026-10-29 14:49:54,167] INFO : Backbone initialization took 0.08s +rank4 [2026-10-29 14:49:54,167] INFO : backbone parameters: 7901376 +rank4 [2026-10-29 14:49:54,167] INFO : Initializing output DNNs +rank4 [2026-10-29 14:49:54,203] INFO : Output DNNs initialization took 0.04s +rank4 [2026-10-29 14:49:54,203] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-29 14:49:54,203] INFO : jagged_attention enabled=False +rank4 [2026-10-29 14:49:54,203] INFO : flash_attn_varlen enabled=False +rank4 [2026-10-29 14:49:54,203] INFO : task_queries enabled=False +rank4 [2026-10-29 14:49:54,203] INFO : classification_norm parameters: 0 +rank4 [2026-10-29 14:49:54,203] INFO : regression_norm parameters: 0 +rank4 [2026-10-29 14:49:54,203] INFO : classification_query parameters: 0 +rank4 [2026-10-29 14:49:54,203] INFO : regression_query parameters: 0 +rank4 [2026-10-29 14:49:54,203] INFO : classification_readout parameters: 0 +rank4 [2026-10-29 14:49:54,203] INFO : regression_readout parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : output_mode=set +rank4 [2026-10-29 14:49:54,204] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-29 14:49:54,204] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_pid parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_pt parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_eta parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-29 14:49:54,204] INFO : nn_energy parameters: 0 +rank4 [2026-10-29 14:49:54,205] INFO : Total MLPF parameters: 12786964 +rank4 [2026-10-29 14:49:54,205] INFO : MLPF __init__ done +rank4 [2026-10-29 14:49:54,205] INFO : Instantiated model +rank4 [2026-10-29 14:49:54,205] INFO : Moving model to device rank=4 +rank4 [2026-10-29 14:49:54,227] INFO : Moved model to device rank=4 +rank4 [2026-10-29 14:49:54,229] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-29 14:49:54,793] INFO : Restoring training from step 29001 +rank4 [2026-10-29 14:49:54,801] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank4 [2026-10-29 14:49:54,801] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-29 14:49:55,513] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank4 [2026-10-29 14:49:55,513] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-29 14:49:55,514] INFO : Trainable parameters: 12786964 +rank4 [2026-10-29 14:49:55,515] INFO : Non-trainable parameters: 23268 +rank4 [2026-10-29 14:49:55,515] INFO : Total parameters: 12810232 +rank4 [2026-10-29 14:49:55,518] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-29 14:49:55,518] INFO : Getting dataloaders +rank4 [2026-10-29 14:49:55,518] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:01,378] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:02,346] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:03,542] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-29 14:50:03,543] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-29 14:50:03,543] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:03,813] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:04,072] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:50:04,326] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-29 14:50:04,326] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-29 14:50:18,083] INFO : Got dataloaders +rank4 [2026-10-29 14:50:18,083] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank4 [2026-10-29 14:50:18,083] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank4 [2026-10-29 14:50:18,083] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank4 [2026-10-29 14:50:18,083] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank4 [2026-10-29 14:50:19,674] INFO : loader split=train rank=4 len=21108 +rank4 [2026-10-29 14:50:19,675] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-29 14:50:19,675] INFO : Starting training from step 29001 to 50000 on rank 4 of 8 +rank4 [2026-10-29 14:50:19,675] INFO : Creating train_iterator +rank4 [2026-10-29 14:50:19,675] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank4 [2026-10-29 14:50:34,044] INFO : Created train_iterator +rank4 [2026-10-29 14:55:12,920] INFO : Step 29100/50000 rank4 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0558 | Regression_eta: 0.0405 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0554 | LR: 4.35e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.7172s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank4 [2026-10-29 14:59:46,789] INFO : Step 29200/50000 rank4 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0529 | Regression_eta: 0.0459 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0543 | LR: 4.33e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3627s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank4 [2026-10-29 15:04:21,834] INFO : Step 29300/50000 rank4 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0639 | Regression_eta: 0.0341 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0563 | LR: 4.30e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0913s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank4 [2026-10-29 15:08:57,032] INFO : Step 29400/50000 rank4 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0615 | Regression_eta: 0.0355 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0366 | LR: 4.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2737s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank4 [2026-10-29 15:13:31,011] INFO : Step 29500/50000 rank4 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0646 | Regression_eta: 0.0412 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0600 | LR: 4.24e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4987s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank4 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank4 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0223 | Classification_binary: 0.1021 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0525 | Regression_eta: 0.0285 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0420 | LR: 4.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5922s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank4 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank4 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0578 | Regression_eta: 0.0431 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0907 | LR: 4.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6726s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank4 [2026-10-29 15:27:18,563] INFO : Step 29800/50000 rank4 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0649 | Regression_eta: 0.0309 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0637 | LR: 4.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6306s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank4 [2026-10-29 15:31:51,911] INFO : Step 29900/50000 rank4 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0540 | Regression_eta: 0.0365 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0460 | LR: 4.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5837s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank4 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank4 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1058 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0457 | Regression_eta: 0.0298 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0439 | LR: 4.11e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4533s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank4 [2026-10-29 15:36:27,739] INFO : Running validation on rank4 +rank4 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 15:37:19,265] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 15:37:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 15:37:51,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 15:37:52,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 15:37:54,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 15:38:04,010] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:38:04,946] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 15:38:20,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 15:38:21,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 15:38:22,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 15:38:23,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 15:38:33,856] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:38:35,222] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 15:38:50,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 15:38:51,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 15:38:52,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 15:38:53,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 15:39:03,275] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank4 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0540 | Regression_eta: 0.0505 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0660 | LR: 4.08e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7105s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank4 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank4 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0609 | Regression_eta: 0.0327 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0565 | LR: 4.06e-04 | DataLoad Time: 0.0265s | Model Forward Time: 4.7631s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank4 [2026-10-29 15:53:38,869] INFO : Step 30300/50000 rank4 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0463 | Regression_eta: 0.0458 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0677 | LR: 4.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3383s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank4 [2026-10-29 15:58:13,389] INFO : Step 30400/50000 rank4 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0542 | Regression_eta: 0.0285 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0363 | LR: 4.00e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank4 [2026-10-29 16:02:51,486] INFO : Step 30500/50000 rank4 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.2246 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0678 | Regression_eta: 0.0563 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0643 | LR: 3.98e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4576s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank4 [2026-10-29 16:07:32,050] INFO : Step 30600/50000 rank4 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0515 | Regression_eta: 0.0412 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0635 | LR: 3.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9375s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank4 [2026-10-29 16:12:18,780] INFO : Step 30700/50000 rank4 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0281 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0519 | Regression_eta: 0.0277 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0595 | LR: 3.92e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2825s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank4 [2026-10-29 16:17:03,636] INFO : Step 30800/50000 rank4 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0568 | Regression_eta: 0.0368 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0753 | LR: 3.90e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0641s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank4 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank4 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0614 | Regression_eta: 0.0368 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0591 | LR: 3.87e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6930s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank4 [2026-10-29 16:26:02,768] INFO : Step 31000/50000 rank4 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0493 | Regression_eta: 0.0269 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0482 | LR: 3.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0249s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank4 [2026-10-29 16:26:49,213] INFO : Running validation on rank4 +rank4 [2026-10-29 16:26:49,213] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:27:39,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:27:40,307] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 16:28:07,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:28:10,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:28:11,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:28:13,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:28:23,362] INFO : Time taken to make predictions on device 4 is: 0.30 min +rank4 [2026-10-29 16:28:24,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:28:24,725] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 16:28:40,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:28:41,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:28:42,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:28:43,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:28:53,919] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 16:28:54,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:28:54,676] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 16:29:10,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:29:11,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:29:11,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:29:12,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:29:22,778] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank4 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0535 | Regression_eta: 0.0375 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0542 | LR: 3.82e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.5520s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank4 [2026-10-29 16:39:13,625] INFO : Step 31200/50000 rank4 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0559 | Regression_eta: 0.0609 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0828 | LR: 3.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0732s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank4 [2026-10-29 16:43:57,174] INFO : Step 31300/50000 rank4 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0682 | Regression_eta: 0.0324 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0633 | LR: 3.76e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8831s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank4 [2026-10-29 16:48:30,667] INFO : Step 31400/50000 rank4 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0564 | Regression_eta: 0.0365 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0480 | LR: 3.74e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7840s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank4 [2026-10-29 16:53:07,752] INFO : Step 31500/50000 rank4 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0454 | Regression_eta: 0.0361 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0630 | LR: 3.71e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank4 [2026-10-29 16:57:38,608] INFO : Step 31600/50000 rank4 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0502 | Regression_eta: 0.0365 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0412 | LR: 3.69e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1679s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank4 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank4 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1200 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0445 | Regression_eta: 0.0312 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0334 | LR: 3.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1012s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank4 [2026-10-29 17:06:47,525] INFO : Step 31800/50000 rank4 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0517 | Regression_eta: 0.0512 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0693 | LR: 3.64e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2351s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank4 [2026-10-29 17:11:27,373] INFO : Step 31900/50000 rank4 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0455 | Regression_eta: 0.0345 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0523 | LR: 3.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5201s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank4 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank4 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0624 | Regression_eta: 0.0458 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0592 | LR: 3.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4958s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank4 [2026-10-29 17:16:12,935] INFO : Running validation on rank4 +rank4 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 17:16:12,935] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 17:17:03,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:17:04,359] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 17:17:32,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 17:17:33,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 17:17:34,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 17:17:35,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 17:17:45,162] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:17:46,494] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 17:18:02,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 17:18:03,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 17:18:04,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 17:18:05,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 17:18:15,372] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 17:18:15,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:18:16,465] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 17:18:32,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 17:18:32,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 17:18:33,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 17:18:34,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 17:18:44,576] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank4 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0516 | Regression_eta: 0.0303 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0507 | LR: 3.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9457s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank4 [2026-10-29 17:28:36,180] INFO : Step 32200/50000 rank4 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0586 | Regression_eta: 0.0350 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0671 | LR: 3.53e-04 | DataLoad Time: 0.0134s | Model Forward Time: 5.4491s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank4 [2026-10-29 17:33:21,418] INFO : Step 32300/50000 rank4 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0178 | Classification_binary: 0.1064 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0548 | Regression_eta: 0.0338 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0410 | LR: 3.51e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.9283s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank4 [2026-10-29 17:38:08,351] INFO : Step 32400/50000 rank4 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0626 | Regression_eta: 0.0325 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0507 | LR: 3.48e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8162s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank4 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank4 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0516 | Regression_eta: 0.0466 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0840 | LR: 3.46e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8573s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank4 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank4 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0579 | Regression_eta: 0.0294 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0424 | LR: 3.43e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.6190s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank4 [2026-10-29 17:52:17,188] INFO : Step 32700/50000 rank4 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0704 | Regression_eta: 0.0504 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0817 | LR: 3.41e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5877s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank4 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank4 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0445 | Regression_eta: 0.0308 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0483 | LR: 3.38e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5879s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank4 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank4 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0516 | Regression_eta: 0.0288 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0600 | LR: 3.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4250s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank4 [2026-10-29 18:06:22,612] INFO : Step 33000/50000 rank4 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0628 | Regression_eta: 0.0339 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0688 | LR: 3.33e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2132s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank4 [2026-10-29 18:06:23,517] INFO : Running validation on rank4 +rank4 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:07:15,766] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 18:07:43,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:07:45,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:07:46,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:07:47,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:07:57,280] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:07:58,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 18:08:14,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:08:15,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:08:16,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:08:17,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:08:27,572] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:08:28,344] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 18:08:43,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:08:44,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:08:45,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:08:46,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:08:56,397] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank4 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0548 | Regression_eta: 0.0282 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0563 | LR: 3.31e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2602s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank4 [2026-10-29 18:18:33,981] INFO : Step 33200/50000 rank4 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0463 | Regression_eta: 0.0440 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0621 | LR: 3.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4346s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank4 [2026-10-29 18:23:15,421] INFO : Step 33300/50000 rank4 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0583 | Regression_eta: 0.0531 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0774 | LR: 3.26e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6026s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank4 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank4 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0483 | Regression_eta: 0.0371 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0397 | LR: 3.23e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.2638s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank4 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank4 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0530 | Regression_eta: 0.0461 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0472 | LR: 3.21e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3717s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank4 [2026-10-29 18:37:27,752] INFO : Step 33600/50000 rank4 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0515 | Regression_eta: 0.0502 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0802 | LR: 3.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3455s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank4 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank4 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1125 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0556 | Regression_eta: 0.0264 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0328 | LR: 3.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3187s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank4 [2026-10-29 18:47:04,985] INFO : Step 33800/50000 rank4 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0411 | Regression_eta: 0.0322 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0418 | LR: 3.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3943s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank4 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank4 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0282 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0441 | Regression_eta: 0.0312 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0641 | LR: 3.11e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1939s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank4 [2026-10-29 18:56:35,807] INFO : Step 34000/50000 rank4 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0467 | Regression_eta: 0.0376 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0521 | LR: 3.09e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1625s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank4 [2026-10-29 18:56:36,806] INFO : Running validation on rank4 +rank4 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:57:29,827] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:57:30,357] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 18:57:54,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:57:55,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:57:57,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:57:58,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:58:08,044] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:58:09,211] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 18:58:25,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:58:26,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:58:27,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:58:28,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:58:38,420] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:58:39,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 18:58:55,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:58:55,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:58:56,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:58:57,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:59:07,682] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 19:04:16,497] INFO : Step 34100/50000 rank4 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.1125 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0486 | Regression_eta: 0.0387 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0393 | LR: 3.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8493s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank4 [2026-10-29 19:09:14,636] INFO : Step 34200/50000 rank4 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0412 | Regression_eta: 0.0343 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0420 | LR: 3.04e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6995s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank4 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank4 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0269 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0547 | Regression_eta: 0.0385 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0497 | LR: 3.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3179s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank4 [2026-10-29 19:18:48,204] INFO : Step 34400/50000 rank4 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0494 | Regression_eta: 0.0407 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0456 | LR: 2.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9246s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank4 [2026-10-29 19:23:49,389] INFO : Step 34500/50000 rank4 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0473 | Regression_eta: 0.0541 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0655 | LR: 2.97e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2321s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank4 [2026-10-29 19:28:34,775] INFO : Step 34600/50000 rank4 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0553 | Regression_eta: 0.0270 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0407 | LR: 2.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4739s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank4 [2026-10-29 19:33:35,744] INFO : Step 34700/50000 rank4 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0561 | Regression_eta: 0.0382 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0433 | LR: 2.92e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1509s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank4 [2026-10-29 19:38:26,589] INFO : Step 34800/50000 rank4 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0473 | Regression_eta: 0.0493 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0876 | LR: 2.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0553s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank4 [2026-10-29 19:43:15,574] INFO : Step 34900/50000 rank4 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0541 | Regression_eta: 0.0281 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0502 | LR: 2.88e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2765s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank4 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank4 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0587 | Regression_eta: 0.0404 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0718 | LR: 2.85e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3869s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank4 [2026-10-29 19:48:21,335] INFO : Running validation on rank4 +rank4 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 19:49:15,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:49:15,989] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 19:49:42,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 19:49:43,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 19:49:44,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 19:49:45,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 19:49:55,668] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:49:56,735] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 19:50:12,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 19:50:13,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 19:50:14,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 19:50:15,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 19:50:25,546] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:50:26,882] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 19:50:42,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 19:50:43,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 19:50:43,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 19:50:44,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 19:50:54,770] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 19:56:07,675] INFO : Step 35100/50000 rank4 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0563 | Regression_eta: 0.0257 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0691 | LR: 2.83e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank4 [2026-10-29 20:01:01,618] INFO : Step 35200/50000 rank4 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0611 | Regression_eta: 0.0446 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0662 | LR: 2.81e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6718s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank4 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank4 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0521 | Regression_eta: 0.0533 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0408 | LR: 2.79e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1442s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank4 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank4 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.0989 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0339 | Regression_eta: 0.0212 | Regression_pt: 0.0357 | Regression_sin_phi: 0.0472 | LR: 2.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4639s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank4 [2026-10-29 20:15:42,289] INFO : Step 35500/50000 rank4 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0426 | Regression_eta: 0.0316 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0469 | LR: 2.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3576s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank4 [2026-10-29 20:20:38,046] INFO : Step 35600/50000 rank4 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0514 | Regression_eta: 0.0493 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0628 | LR: 2.72e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1897s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank4 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank4 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1179 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0536 | Regression_eta: 0.0315 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0446 | LR: 2.70e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank4 [2026-10-29 20:30:22,091] INFO : Step 35800/50000 rank4 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0483 | Regression_eta: 0.0418 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0544 | LR: 2.68e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.2792s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank4 [2026-10-29 20:35:20,189] INFO : Step 35900/50000 rank4 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0636 | Regression_eta: 0.0341 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0592 | LR: 2.65e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6692s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank4 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank4 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0455 | Regression_eta: 0.0320 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0557 | LR: 2.63e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2592s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank4 [2026-10-29 20:40:17,059] INFO : Running validation on rank4 +rank4 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 20:41:08,401] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:41:08,960] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 20:41:36,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 20:41:37,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 20:41:38,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 20:41:39,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 20:41:49,185] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:41:50,339] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 20:42:06,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 20:42:07,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 20:42:08,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 20:42:09,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 20:42:19,262] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:42:20,693] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 20:42:36,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 20:42:36,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 20:42:37,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 20:42:38,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 20:42:48,570] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 20:47:46,989] INFO : Step 36100/50000 rank4 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0492 | Regression_eta: 0.0283 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0341 | LR: 2.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4496s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank4 [2026-10-29 20:52:22,841] INFO : Step 36200/50000 rank4 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0464 | Regression_eta: 0.0338 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0456 | LR: 2.59e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3970s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank4 [2026-10-29 20:56:59,585] INFO : Step 36300/50000 rank4 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0669 | Regression_eta: 0.0330 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0475 | LR: 2.57e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2284s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank4 [2026-10-29 21:01:38,353] INFO : Step 36400/50000 rank4 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0577 | Regression_eta: 0.0281 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0637 | LR: 2.55e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8769s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank4 [2026-10-29 21:06:14,246] INFO : Step 36500/50000 rank4 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0457 | Regression_eta: 0.0344 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0375 | LR: 2.52e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4231s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank4 [2026-10-29 21:10:49,732] INFO : Step 36600/50000 rank4 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0339 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0544 | Regression_eta: 0.0299 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0511 | LR: 2.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2769s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank4 [2026-10-29 21:15:35,109] INFO : Step 36700/50000 rank4 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0481 | Regression_eta: 0.0649 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0492 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2698s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank4 [2026-10-29 21:20:28,004] INFO : Step 36800/50000 rank4 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0581 | Regression_eta: 0.0280 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0327 | LR: 2.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4008s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank4 [2026-10-29 21:25:23,682] INFO : Step 36900/50000 rank4 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0413 | Regression_eta: 0.0310 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0756 | LR: 2.44e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9352s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank4 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank4 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0434 | Regression_eta: 0.0363 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0434 | LR: 2.42e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9280s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank4 [2026-10-29 21:30:17,323] INFO : Running validation on rank4 +rank4 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:31:08,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:31:08,622] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 21:31:35,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:31:36,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:31:37,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:31:38,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:31:48,549] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:31:49,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 21:32:05,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:32:06,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:32:07,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:32:08,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:32:18,748] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:32:20,812] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 21:32:36,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:32:37,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:32:38,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:32:38,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:32:48,926] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 21:37:56,198] INFO : Step 37100/50000 rank4 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1163 | Regression_cos_phi: 0.1047 | Regression_energy: 0.0499 | Regression_eta: 0.0476 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0639 | LR: 2.40e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6912s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank4 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank4 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0418 | Regression_eta: 0.0323 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0594 | LR: 2.38e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6958s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank4 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank4 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0455 | Regression_eta: 0.0280 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0570 | LR: 2.36e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5595s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank4 [2026-10-29 21:52:56,910] INFO : Step 37400/50000 rank4 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0274 | Classification_binary: 0.1048 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0435 | Regression_eta: 0.0306 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0244 | LR: 2.34e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6116s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank4 [2026-10-29 21:57:43,877] INFO : Step 37500/50000 rank4 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0567 | Regression_eta: 0.0638 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0566 | LR: 2.32e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7098s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank4 [2026-10-29 22:02:29,840] INFO : Step 37600/50000 rank4 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0422 | Regression_eta: 0.0277 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0412 | LR: 2.30e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2476s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank4 [2026-10-29 22:07:05,609] INFO : Step 37700/50000 rank4 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0439 | Regression_eta: 0.0334 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0398 | LR: 2.28e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1554s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank4 [2026-10-29 22:11:38,775] INFO : Step 37800/50000 rank4 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1029 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0506 | Regression_eta: 0.0300 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0399 | LR: 2.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2587s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank4 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank4 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0510 | Regression_eta: 0.0307 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0619 | LR: 2.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1253s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank4 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank4 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0352 | Regression_eta: 0.0451 | Regression_pt: 0.0362 | Regression_sin_phi: 0.0437 | LR: 2.22e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2884s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank4 [2026-10-29 22:20:57,876] INFO : Running validation on rank4 +rank4 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 22:21:48,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:21:48,921] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 22:22:15,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 22:22:16,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 22:22:17,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 22:22:18,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 22:22:28,316] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 22:22:28,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:22:29,427] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 22:22:45,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 22:22:46,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 22:22:47,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 22:22:48,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 22:22:58,212] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:22:59,393] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 22:23:14,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 22:23:15,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 22:23:16,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 22:23:17,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 22:23:27,204] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank4 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0380 | Regression_eta: 0.0305 | Regression_pt: 0.0381 | Regression_sin_phi: 0.0341 | LR: 2.20e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2309s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank4 [2026-10-29 22:33:07,241] INFO : Step 38200/50000 rank4 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0389 | Regression_eta: 0.0315 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0643 | LR: 2.18e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7608s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank4 [2026-10-29 22:37:42,242] INFO : Step 38300/50000 rank4 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0281 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0429 | Regression_eta: 0.0289 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0424 | LR: 2.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5927s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank4 [2026-10-29 22:42:17,609] INFO : Step 38400/50000 rank4 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.0988 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0544 | Regression_eta: 0.0381 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0462 | LR: 2.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4882s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank4 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank4 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0407 | Regression_eta: 0.0244 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0429 | LR: 2.12e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5213s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank4 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank4 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0435 | Regression_eta: 0.0264 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0461 | LR: 2.11e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.7274s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank4 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank4 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0475 | Regression_eta: 0.0334 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0511 | LR: 2.09e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1557s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank4 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank4 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0547 | Regression_eta: 0.0338 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0564 | LR: 2.07e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3994s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank4 [2026-10-29 23:07:29,432] INFO : Step 38900/50000 rank4 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0473 | Regression_eta: 0.0261 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0571 | LR: 2.05e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.8955s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank4 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank4 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0454 | Regression_eta: 0.0264 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0414 | LR: 2.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4704s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank4 [2026-10-29 23:12:38,865] INFO : Running validation on rank4 +rank4 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 23:13:33,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:13:34,228] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-29 23:14:06,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 23:14:07,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 23:14:08,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 23:14:09,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 23:14:19,618] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 23:14:20,234] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:14:20,782] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-29 23:14:37,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 23:14:38,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 23:14:38,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 23:14:39,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 23:14:49,970] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:14:50,899] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-29 23:15:06,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 23:15:07,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 23:15:08,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 23:15:09,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 23:15:19,155] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank4 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1179 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0563 | Regression_eta: 0.0314 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0481 | LR: 2.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9863s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank4 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank4 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0483 | Regression_eta: 0.0357 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0534 | LR: 2.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5917s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank4 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank4 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0244 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0402 | Regression_eta: 0.0341 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0457 | LR: 1.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6306s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank4 [2026-10-29 23:34:45,034] INFO : Step 39400/50000 rank4 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0459 | Regression_eta: 0.0335 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0384 | LR: 1.96e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.6763s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank4 [2026-10-29 23:39:27,723] INFO : Step 39500/50000 rank4 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0357 | Regression_eta: 0.0268 | Regression_pt: 0.0370 | Regression_sin_phi: 0.0334 | LR: 1.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4393s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank4 [2026-10-29 23:43:57,879] INFO : Step 39600/50000 rank4 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0501 | Regression_eta: 0.0401 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0621 | LR: 1.93e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4682s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank4 [2026-10-29 23:48:42,956] INFO : Step 39700/50000 rank4 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1185 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0538 | Regression_eta: 0.0280 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0608 | LR: 1.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4041s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank4 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank4 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0234 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0437 | Regression_eta: 0.0311 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0304 | LR: 1.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5396s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank4 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank4 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0413 | Regression_eta: 0.0442 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0497 | LR: 1.88e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7674s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank4 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank4 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1069 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0450 | Regression_eta: 0.0236 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0311 | LR: 1.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9273s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank4 [2026-10-30 00:05:56,735] INFO : Running validation on rank4 +rank4 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:06:50,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:06:51,352] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 00:07:17,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:07:18,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:07:19,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:07:20,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:07:30,922] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:07:31,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 00:07:47,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:07:48,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:07:49,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:07:50,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:08:00,583] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:08:01,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 00:08:17,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 00:08:18,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 00:08:19,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 00:08:20,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 00:08:30,019] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank4 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0343 | Regression_eta: 0.0344 | Regression_pt: 0.0358 | Regression_sin_phi: 0.0383 | LR: 1.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6423s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank4 [2026-10-30 00:18:43,888] INFO : Step 40200/50000 rank4 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0278 | Classification_binary: 0.1135 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0472 | Regression_eta: 0.0219 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0317 | LR: 1.83e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4453s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank4 [2026-10-30 00:23:35,703] INFO : Step 40300/50000 rank4 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0578 | Regression_eta: 0.0285 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0811 | LR: 1.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5331s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank4 [2026-10-30 00:28:38,368] INFO : Step 40400/50000 rank4 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1071 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0466 | Regression_eta: 0.0282 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0486 | LR: 1.79e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2217s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank4 [2026-10-30 00:33:25,152] INFO : Step 40500/50000 rank4 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1078 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0409 | Regression_eta: 0.0199 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0237 | LR: 1.78e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2268s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank4 [2026-10-30 00:38:07,831] INFO : Step 40600/50000 rank4 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0409 | Regression_eta: 0.0274 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0462 | LR: 1.76e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2965s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank4 [2026-10-30 00:42:54,582] INFO : Step 40700/50000 rank4 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0309 | Regression_energy: 0.0537 | Regression_eta: 0.0263 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0481 | LR: 1.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2535s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank4 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank4 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0282 | Classification_binary: 0.0978 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0393 | Regression_eta: 0.0304 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0428 | LR: 1.73e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8893s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank4 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank4 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0432 | Regression_eta: 0.0327 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0662 | LR: 1.72e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7100s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank4 [2026-10-30 00:57:38,928] INFO : Step 41000/50000 rank4 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0388 | Regression_eta: 0.0242 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0454 | LR: 1.70e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3871s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank4 [2026-10-30 00:57:39,931] INFO : Running validation on rank4 +rank4 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:58:31,732] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:58:32,294] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 00:58:59,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:59:00,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:59:01,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:59:02,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:59:12,404] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:59:13,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:59:13,671] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 00:59:29,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:59:30,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:59:31,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:59:32,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:59:42,690] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:59:44,018] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 00:59:59,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:00:00,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:00:01,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:00:02,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:00:12,268] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 01:05:10,995] INFO : Step 41100/50000 rank4 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1098 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0378 | Regression_eta: 0.0223 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0537 | LR: 1.69e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7647s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank4 [2026-10-30 01:10:09,652] INFO : Step 41200/50000 rank4 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1120 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0653 | Regression_eta: 0.0318 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0601 | LR: 1.67e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6353s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank4 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank4 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0490 | Regression_eta: 0.0246 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0523 | LR: 1.66e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2626s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank4 [2026-10-30 01:19:55,522] INFO : Step 41400/50000 rank4 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0407 | Regression_eta: 0.0229 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0386 | LR: 1.64e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.6717s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank4 [2026-10-30 01:24:55,608] INFO : Step 41500/50000 rank4 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.0933 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0411 | Regression_eta: 0.0250 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0564 | LR: 1.63e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4514s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank4 [2026-10-30 01:29:39,760] INFO : Step 41600/50000 rank4 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0447 | Regression_eta: 0.0265 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0373 | LR: 1.61e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0157s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank4 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank4 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0356 | Regression_eta: 0.0314 | Regression_pt: 0.0367 | Regression_sin_phi: 0.0385 | LR: 1.60e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7038s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank4 [2026-10-30 01:39:10,939] INFO : Step 41800/50000 rank4 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0378 | Regression_eta: 0.0496 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0504 | LR: 1.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8802s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank4 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank4 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0428 | Regression_eta: 0.0321 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0526 | LR: 1.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2475s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank4 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank4 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0378 | Regression_eta: 0.0393 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0554 | LR: 1.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2693s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank4 [2026-10-30 01:49:03,869] INFO : Running validation on rank4 +rank4 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 01:49:03,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 01:49:57,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 01:50:24,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 01:50:26,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 01:50:27,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 01:50:28,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 01:50:38,109] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:50:39,486] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 01:50:55,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 01:50:56,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 01:50:57,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 01:50:58,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 01:51:08,040] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:51:10,100] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 01:51:26,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:51:26,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:51:27,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:51:28,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:51:38,654] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank4 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0555 | Regression_eta: 0.0432 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0497 | LR: 1.54e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0041s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank4 [2026-10-30 02:01:47,473] INFO : Step 42200/50000 rank4 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0531 | Regression_eta: 0.0323 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0349 | LR: 1.53e-04 | DataLoad Time: 0.0168s | Model Forward Time: 7.2722s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank4 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank4 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-30 02:06:41,297] INFO : Step 42300/50000 rank4 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1171 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0459 | Regression_eta: 0.0187 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0389 | LR: 1.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8843s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank4 [2026-10-30 02:11:46,438] INFO : Step 42400/50000 rank4 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1049 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0420 | Regression_eta: 0.0317 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0549 | LR: 1.50e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1608s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank4 [2026-10-30 02:16:38,986] INFO : Step 42500/50000 rank4 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0388 | Regression_eta: 0.0349 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0518 | LR: 1.49e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8615s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank4 [2026-10-30 02:21:27,109] INFO : Step 42600/50000 rank4 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0666 | Regression_eta: 0.0371 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0737 | LR: 1.48e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.3544s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank4 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank4 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0500 | Regression_eta: 0.0229 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0338 | LR: 1.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6233s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank4 [2026-10-30 02:31:38,796] INFO : Step 42800/50000 rank4 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1126 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0362 | Regression_eta: 0.0309 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0425 | LR: 1.45e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6410s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank4 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank4 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0452 | Regression_eta: 0.0218 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0483 | LR: 1.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8539s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank4 [2026-10-30 02:41:13,873] INFO : Step 43000/50000 rank4 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1091 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0464 | Regression_eta: 0.0327 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0648 | LR: 1.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5958s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank4 [2026-10-30 02:41:14,878] INFO : Running validation on rank4 +rank4 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 02:42:10,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 02:42:38,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 02:42:39,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 02:42:40,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 02:42:41,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 02:42:51,368] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 02:42:51,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:42:52,443] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 02:43:08,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 02:43:09,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 02:43:10,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 02:43:11,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 02:43:21,680] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 02:43:38,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 02:43:39,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 02:43:40,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 02:43:41,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 02:43:51,240] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 02:48:51,376] INFO : Step 43100/50000 rank4 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0410 | Regression_eta: 0.0307 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0399 | LR: 1.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9461s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank4 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank4 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0459 | Regression_eta: 0.0354 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0512 | LR: 1.40e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5633s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank4 [2026-10-30 02:58:07,502] INFO : Step 43300/50000 rank4 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1145 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0477 | Regression_eta: 0.0247 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0469 | LR: 1.39e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0496s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank4 [2026-10-30 03:02:42,808] INFO : Step 43400/50000 rank4 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1046 | Regression_cos_phi: 0.0223 | Regression_energy: 0.0397 | Regression_eta: 0.0226 | Regression_pt: 0.0409 | Regression_sin_phi: 0.0330 | LR: 1.38e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4745s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank4 [2026-10-30 03:07:16,804] INFO : Step 43500/50000 rank4 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0405 | Regression_eta: 0.0300 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0379 | LR: 1.37e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9335s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank4 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank4 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0429 | Regression_eta: 0.0206 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0321 | LR: 1.36e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8423s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank4 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank4 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0666 | Regression_eta: 0.0274 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0498 | LR: 1.35e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.1374s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank4 [2026-10-30 03:21:10,515] INFO : Step 43800/50000 rank4 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0455 | Regression_eta: 0.0360 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0672 | LR: 1.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2558s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank4 [2026-10-30 03:26:01,004] INFO : Step 43900/50000 rank4 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0474 | Regression_eta: 0.0312 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0619 | LR: 1.33e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.9702s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank4 [2026-10-30 03:30:36,933] INFO : Step 44000/50000 rank4 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0188 | Classification_binary: 0.0948 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0301 | Regression_eta: 0.0513 | Regression_pt: 0.0305 | Regression_sin_phi: 0.0399 | LR: 1.32e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8179s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank4 [2026-10-30 03:30:37,938] INFO : Running validation on rank4 +rank4 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 03:31:28,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:31:29,164] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 03:31:57,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 03:31:58,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 03:31:59,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 03:32:00,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 03:32:10,840] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 03:32:27,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 03:32:28,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 03:32:29,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 03:32:30,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 03:32:40,914] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:32:41,628] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 03:32:57,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 03:32:58,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 03:32:58,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 03:32:59,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 03:33:09,672] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 03:38:15,294] INFO : Step 44100/50000 rank4 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0439 | Regression_eta: 0.0264 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0361 | LR: 1.31e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7484s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank4 [2026-10-30 03:43:04,522] INFO : Step 44200/50000 rank4 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0477 | Regression_eta: 0.0353 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0541 | LR: 1.30e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank4 [2026-10-30 03:47:46,464] INFO : Step 44300/50000 rank4 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0459 | Regression_eta: 0.0231 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0509 | LR: 1.29e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4332s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank4 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank4 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.0962 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0460 | Regression_eta: 0.0256 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0270 | LR: 1.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3364s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank4 [2026-10-30 03:57:11,972] INFO : Step 44500/50000 rank4 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1264 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0431 | Regression_eta: 0.0293 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0369 | LR: 1.27e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3176s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank4 [2026-10-30 04:01:48,197] INFO : Step 44600/50000 rank4 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0500 | Regression_eta: 0.0410 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0429 | LR: 1.26e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5958s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank4 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank4 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0601 | Regression_eta: 0.0238 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0389 | LR: 1.25e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6223s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank4 [2026-10-30 04:11:07,220] INFO : Step 44800/50000 rank4 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.0912 | Regression_cos_phi: 0.0307 | Regression_energy: 0.0397 | Regression_eta: 0.0205 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0443 | LR: 1.24e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5868s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank4 [2026-10-30 04:15:54,537] INFO : Step 44900/50000 rank4 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0433 | Regression_eta: 0.0330 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0878 | LR: 1.23e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6719s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank4 [2026-10-30 04:20:36,055] INFO : Step 45000/50000 rank4 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0381 | Regression_eta: 0.0271 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0424 | LR: 1.22e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7883s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank4 [2026-10-30 04:20:37,109] INFO : Running validation on rank4 +rank4 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 04:21:27,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:21:28,560] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 04:21:56,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 04:21:57,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 04:21:58,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 04:21:59,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 04:22:09,900] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:22:10,892] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 04:22:26,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 04:22:27,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 04:22:28,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 04:22:29,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 04:22:39,483] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:22:41,054] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 04:22:56,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 04:22:57,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 04:22:58,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 04:22:59,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 04:23:09,315] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank4 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1070 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0450 | Regression_eta: 0.0508 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0423 | LR: 1.21e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2107s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank4 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank4 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0444 | Regression_eta: 0.0329 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0510 | LR: 1.20e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank4 [2026-10-30 04:37:52,085] INFO : Step 45300/50000 rank4 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0442 | Regression_eta: 0.0309 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0586 | LR: 1.19e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8150s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank4 [2026-10-30 04:42:38,426] INFO : Step 45400/50000 rank4 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0340 | Regression_energy: 0.0435 | Regression_eta: 0.0264 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0331 | LR: 1.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4016s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank4 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank4 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1096 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0522 | Regression_eta: 0.0282 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0436 | LR: 1.18e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3867s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank4 [2026-10-30 04:52:16,191] INFO : Step 45600/50000 rank4 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0408 | Regression_eta: 0.0391 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0546 | LR: 1.17e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4808s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank4 [2026-10-30 04:57:07,844] INFO : Step 45700/50000 rank4 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0220 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0447 | Regression_eta: 0.0356 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0619 | LR: 1.16e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4875s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank4 [2026-10-30 05:01:56,082] INFO : Step 45800/50000 rank4 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1186 | Regression_cos_phi: 0.0342 | Regression_energy: 0.0632 | Regression_eta: 0.0241 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0329 | LR: 1.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6474s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank4 [2026-10-30 05:06:42,062] INFO : Step 45900/50000 rank4 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0222 | Classification_binary: 0.1003 | Regression_cos_phi: 0.0291 | Regression_energy: 0.0348 | Regression_eta: 0.0254 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0295 | LR: 1.15e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2870s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank4 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank4 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1069 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0454 | Regression_eta: 0.0217 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0323 | LR: 1.14e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.4363s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank4 [2026-10-30 05:11:37,037] INFO : Running validation on rank4 +rank4 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 05:12:27,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:12:27,993] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 05:12:53,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 05:12:54,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 05:12:55,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 05:12:56,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 05:13:06,169] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 05:13:22,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 05:13:23,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 05:13:24,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 05:13:25,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 05:13:35,702] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 05:13:36,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 05:13:52,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 05:13:53,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 05:13:54,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 05:13:55,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 05:14:05,330] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 05:19:00,536] INFO : Step 46100/50000 rank4 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.0990 | Regression_cos_phi: 0.0291 | Regression_energy: 0.0536 | Regression_eta: 0.0254 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0390 | LR: 1.13e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5898s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank4 [2026-10-30 05:23:34,446] INFO : Step 46200/50000 rank4 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0389 | Regression_eta: 0.0324 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0506 | LR: 1.13e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7915s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank4 [2026-10-30 05:28:08,608] INFO : Step 46300/50000 rank4 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0486 | Regression_eta: 0.0325 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0485 | LR: 1.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9763s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank4 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank4 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0485 | Regression_eta: 0.0287 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0408 | LR: 1.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7887s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank4 [2026-10-30 05:37:21,831] INFO : Step 46500/50000 rank4 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0513 | Regression_eta: 0.0382 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0347 | LR: 1.11e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank4 [2026-10-30 05:41:57,343] INFO : Step 46600/50000 rank4 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0367 | Regression_eta: 0.0324 | Regression_pt: 0.0371 | Regression_sin_phi: 0.0402 | LR: 1.10e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.7864s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank4 [2026-10-30 05:46:32,411] INFO : Step 46700/50000 rank4 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0664 | Regression_eta: 0.0258 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0453 | LR: 1.10e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6533s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank4 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank4 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.0978 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0444 | Regression_eta: 0.0256 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0437 | LR: 1.09e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7027s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank4 [2026-10-30 05:55:41,597] INFO : Step 46900/50000 rank4 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0415 | Regression_eta: 0.0332 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0537 | LR: 1.09e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7515s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank4 [2026-10-30 06:00:20,615] INFO : Step 47000/50000 rank4 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0469 | Regression_eta: 0.0408 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0402 | LR: 1.08e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8054s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank4 [2026-10-30 06:00:21,750] INFO : Running validation on rank4 +rank4 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 06:01:12,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 06:01:37,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 06:01:38,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 06:01:39,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 06:01:40,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 06:01:50,577] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:01:51,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 06:02:07,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 06:02:08,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 06:02:09,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 06:02:10,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 06:02:20,482] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:02:21,870] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 06:02:37,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 06:02:38,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 06:02:39,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 06:02:40,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 06:02:50,173] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 06:07:51,196] INFO : Step 47100/50000 rank4 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0372 | Regression_eta: 0.0300 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0384 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6264s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank4 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank4 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0368 | Regression_eta: 0.0301 | Regression_pt: 0.0383 | Regression_sin_phi: 0.0553 | LR: 1.07e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9891s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank4 [2026-10-30 06:17:04,150] INFO : Step 47300/50000 rank4 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0257 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0370 | Regression_eta: 0.0362 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0461 | LR: 1.06e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0311s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank4 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank4 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0506 | Regression_eta: 0.0231 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0382 | LR: 1.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4530s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank4 [2026-10-30 06:26:15,801] INFO : Step 47500/50000 rank4 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.0887 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0470 | Regression_eta: 0.0234 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0368 | LR: 1.06e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6770s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank4 [2026-10-30 06:30:50,119] INFO : Step 47600/50000 rank4 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1311 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0429 | Regression_eta: 0.0479 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0522 | LR: 1.05e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9490s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank4 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank4 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1185 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0341 | Regression_eta: 0.0263 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0305 | LR: 1.05e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9108s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank4 [2026-10-30 06:40:00,408] INFO : Step 47800/50000 rank4 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0195 | Classification_binary: 0.0961 | Regression_cos_phi: 0.0272 | Regression_energy: 0.0313 | Regression_eta: 0.0258 | Regression_pt: 0.0322 | Regression_sin_phi: 0.0477 | LR: 1.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7813s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank4 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank4 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0488 | Regression_eta: 0.0473 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0572 | LR: 1.04e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8495s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank4 [2026-10-30 06:49:09,637] INFO : Step 48000/50000 rank4 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0361 | Regression_eta: 0.0374 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0436 | LR: 1.04e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8826s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank4 [2026-10-30 06:49:10,726] INFO : Running validation on rank4 +rank4 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 06:50:01,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 06:50:27,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 06:50:28,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 06:50:29,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 06:50:30,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 06:50:40,503] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:50:41,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:50:41,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 06:50:57,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 06:50:58,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 06:50:59,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 06:51:00,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 06:51:10,611] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:51:11,431] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 06:51:26,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 06:51:27,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 06:51:28,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 06:51:29,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 06:51:39,451] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank4 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0262 | Classification_binary: 0.1049 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0400 | Regression_eta: 0.0211 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0372 | LR: 1.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6270s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank4 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank4 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0361 | Regression_eta: 0.0287 | Regression_pt: 0.0378 | Regression_sin_phi: 0.0411 | LR: 1.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.3156s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank4 [2026-10-30 07:05:53,464] INFO : Step 48300/50000 rank4 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0468 | Regression_eta: 0.0266 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0447 | LR: 1.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1166s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank4 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank4 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1070 | Regression_cos_phi: 0.0346 | Regression_energy: 0.0503 | Regression_eta: 0.0247 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0383 | LR: 1.02e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8692s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank4 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank4 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0421 | Regression_eta: 0.0428 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0619 | LR: 1.02e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.9636s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank4 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank4 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0389 | Regression_eta: 0.0271 | Regression_pt: 0.0408 | Regression_sin_phi: 0.0406 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8065s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank4 [2026-10-30 07:24:30,058] INFO : Step 48700/50000 rank4 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0268 | Classification_binary: 0.1221 | Regression_cos_phi: 0.0294 | Regression_energy: 0.0391 | Regression_eta: 0.0225 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0442 | LR: 1.02e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9975s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank4 [2026-10-30 07:29:10,217] INFO : Step 48800/50000 rank4 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1073 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0352 | Regression_eta: 0.0288 | Regression_pt: 0.0360 | Regression_sin_phi: 0.0465 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9229s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank4 [2026-10-30 07:33:48,297] INFO : Step 48900/50000 rank4 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0426 | Regression_eta: 0.0350 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0559 | LR: 1.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7885s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank4 [2026-10-30 07:38:22,689] INFO : Step 49000/50000 rank4 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0420 | Regression_eta: 0.0307 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0466 | LR: 1.01e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8907s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank4 [2026-10-30 07:38:23,699] INFO : Running validation on rank4 +rank4 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:39:14,632] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 07:39:39,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 07:39:40,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 07:39:41,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 07:39:42,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 07:39:52,660] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 07:39:53,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:39:53,779] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 07:40:09,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 07:40:10,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 07:40:11,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 07:40:12,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 07:40:22,684] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:40:24,011] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 07:40:39,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 07:40:40,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 07:40:41,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 07:40:42,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 07:40:52,169] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank4 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0192 | Classification_binary: 0.1154 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0450 | Regression_eta: 0.0511 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0577 | LR: 1.01e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.6572s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank4 [2026-10-30 07:50:30,114] INFO : Step 49200/50000 rank4 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1039 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0474 | Regression_eta: 0.0231 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0424 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5153s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank4 [2026-10-30 07:55:07,372] INFO : Step 49300/50000 rank4 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0490 | Regression_eta: 0.0364 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0495 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8692s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank4 [2026-10-30 07:59:43,351] INFO : Step 49400/50000 rank4 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0661 | Regression_eta: 0.0381 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0468 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8899s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank4 [2026-10-30 08:04:24,759] INFO : Step 49500/50000 rank4 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1236 | Regression_cos_phi: 0.0335 | Regression_energy: 0.0495 | Regression_eta: 0.0240 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0305 | LR: 1.00e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3367s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank4 [2026-10-30 08:08:58,924] INFO : Step 49600/50000 rank4 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.1231 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0346 | Regression_eta: 0.0234 | Regression_pt: 0.0348 | Regression_sin_phi: 0.0371 | LR: 1.00e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8770s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank4 [2026-10-30 08:13:36,933] INFO : Step 49700/50000 rank4 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0312 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0410 | Regression_eta: 0.0205 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0324 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7051s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank4 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank4 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.1001 | Regression_cos_phi: 0.0283 | Regression_energy: 0.0363 | Regression_eta: 0.0216 | Regression_pt: 0.0375 | Regression_sin_phi: 0.0337 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5264s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank4 [2026-10-30 08:22:53,417] INFO : Step 49900/50000 rank4 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0502 | Regression_eta: 0.0368 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0467 | LR: 1.00e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2035s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank4 [2026-10-30 08:27:37,250] INFO : Step 50000/50000 rank4 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0408 | Regression_eta: 0.0242 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0380 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.3463s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank4 [2026-10-30 08:27:38,187] INFO : Running validation on rank4 +rank4 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 08:28:29,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 08:28:29,585] INFO : Running predictions on clic_edm_ttbar_hits +rank4 [2026-10-30 08:29:00,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 08:29:01,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 08:29:02,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 08:29:03,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 08:29:13,312] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 08:29:13,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 08:29:14,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank4 [2026-10-30 08:29:30,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 08:29:32,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 08:29:33,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 08:29:34,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 08:29:44,311] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 08:29:46,086] INFO : Running predictions on clic_edm_qq_hits +rank4 [2026-10-30 08:30:01,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 08:30:02,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 08:30:03,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 08:30:04,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 08:30:14,142] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 08:30:15,208] INFO : Training completed. Total time on device 4: 1059.926min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.5 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..4587d308a1a12a48d1a375fc3bb511878efec622 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.5 @@ -0,0 +1,2654 @@ +rank5 [2026-10-28 11:12:32,393] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-28 11:12:32,394] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank5 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank5 [2026-10-28 11:12:57,209] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-28 11:12:57,209] INFO : Initializing backbone layer 0 +rank5 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank5 [2026-10-28 11:12:57,254] INFO : Initializing backbone layer 2 +rank5 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 3 +rank5 [2026-10-28 11:12:57,272] INFO : Initializing backbone layer 4 +rank5 [2026-10-28 11:12:57,280] INFO : Initializing backbone layer 5 +rank5 [2026-10-28 11:12:57,289] INFO : Backbone initialization took 0.08s +rank5 [2026-10-28 11:12:57,289] INFO : backbone parameters: 7901376 +rank5 [2026-10-28 11:12:57,290] INFO : Initializing output DNNs +rank5 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank5 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank5 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank5 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank5 [2026-10-28 11:12:57,329] INFO : classification_norm parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : regression_norm parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : classification_query parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : regression_query parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : classification_readout parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : regression_readout parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : output_mode=set +rank5 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank5 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank5 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank5 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank5 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=5 +rank5 [2026-10-28 11:12:57,366] INFO : Moved model to device rank=5 +rank5 [2026-10-28 11:12:57,368] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-28 11:12:57,368] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-28 11:12:57,855] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank5 [2026-10-28 11:12:57,855] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-28 11:12:57,856] INFO : Trainable parameters: 12786964 +rank5 [2026-10-28 11:12:57,856] INFO : Non-trainable parameters: 23268 +rank5 [2026-10-28 11:12:57,856] INFO : Total parameters: 12810232 +rank5 [2026-10-28 11:12:57,859] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-28 11:12:57,859] INFO : Getting dataloaders +rank5 [2026-10-28 11:12:57,860] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:03,725] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:04,755] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,000] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-28 11:13:06,001] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-28 11:13:06,001] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,271] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,530] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,794] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-28 11:13:06,794] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-28 11:13:20,902] INFO : Got dataloaders +rank5 [2026-10-28 11:13:20,902] INFO : loader split=train rank=5 len=21108 +rank5 [2026-10-28 11:13:20,902] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-28 11:13:20,903] INFO : Starting training from step 1 to 50000 on rank 5 of 8 +rank5 [2026-10-28 11:13:20,903] INFO : Creating train_iterator +rank5 [2026-10-28 11:13:20,903] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-28 11:13:35,396] INFO : Created train_iterator +rank5 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank5 [2026-10-28 11:18:16,510] INFO : Step 100/50000 rank5 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8240 | Classification_binary: 0.4712 | Regression_cos_phi: 0.0074 | Regression_energy: 0.8421 | Regression_eta: 0.0167 | Regression_pt: 0.8277 | Regression_sin_phi: 0.0074 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 3.6170s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank5 [2026-10-28 11:22:47,962] INFO : Step 200/50000 rank5 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3606 | Classification_binary: 0.5584 | Regression_cos_phi: 0.3261 | Regression_energy: 0.4198 | Regression_eta: 0.2567 | Regression_pt: 0.4268 | Regression_sin_phi: 0.3778 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.4381s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank5 [2026-10-28 11:27:17,892] INFO : Step 300/50000 rank5 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3030 | Classification_binary: 0.4859 | Regression_cos_phi: 0.3239 | Regression_energy: 0.3170 | Regression_eta: 0.2497 | Regression_pt: 0.3207 | Regression_sin_phi: 0.3483 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 4.3674s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank5 [2026-10-28 11:31:48,894] INFO : Step 400/50000 rank5 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2730 | Classification_binary: 0.4287 | Regression_cos_phi: 0.3433 | Regression_energy: 0.3087 | Regression_eta: 0.2305 | Regression_pt: 0.3038 | Regression_sin_phi: 0.2676 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 4.2442s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank5 [2026-10-28 11:36:20,164] INFO : Step 500/50000 rank5 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2344 | Classification_binary: 0.5100 | Regression_cos_phi: 0.2645 | Regression_energy: 0.3019 | Regression_eta: 0.3025 | Regression_pt: 0.3160 | Regression_sin_phi: 0.2950 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank5 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank5 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2359 | Classification_binary: 0.4778 | Regression_cos_phi: 0.3294 | Regression_energy: 0.2895 | Regression_eta: 0.2318 | Regression_pt: 0.2934 | Regression_sin_phi: 0.3156 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 4.5136s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank5 [2026-10-28 11:45:32,572] INFO : Step 700/50000 rank5 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2023 | Classification_binary: 0.4004 | Regression_cos_phi: 0.2120 | Regression_energy: 0.2415 | Regression_eta: 0.1912 | Regression_pt: 0.2477 | Regression_sin_phi: 0.1937 | LR: 1.00e-03 | DataLoad Time: 0.0186s | Model Forward Time: 3.7389s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank5 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank5 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2124 | Classification_binary: 0.4075 | Regression_cos_phi: 0.1853 | Regression_energy: 0.3130 | Regression_eta: 0.1637 | Regression_pt: 0.3149 | Regression_sin_phi: 0.2144 | LR: 9.99e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.2806s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank5 [2026-10-28 11:54:58,536] INFO : Step 900/50000 rank5 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2083 | Classification_binary: 0.4988 | Regression_cos_phi: 0.2977 | Regression_energy: 0.2946 | Regression_eta: 0.1925 | Regression_pt: 0.3009 | Regression_sin_phi: 0.2325 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4571s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank5 [2026-10-28 11:59:55,250] INFO : Step 1000/50000 rank5 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1849 | Classification_binary: 0.4176 | Regression_cos_phi: 0.2172 | Regression_energy: 0.2793 | Regression_eta: 0.1391 | Regression_pt: 0.2824 | Regression_sin_phi: 0.1893 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.7519s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank5 [2026-10-28 11:59:56,332] INFO : Running validation on rank5 +rank5 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 12:00:48,695] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:00:49,300] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 12:01:16,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:01:17,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:01:18,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:01:19,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:01:29,771] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:01:30,838] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 12:01:46,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:01:47,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:01:48,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:01:49,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:01:59,757] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 12:02:00,466] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:02:01,007] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 12:02:16,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:02:17,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:02:17,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:02:18,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:02:28,755] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 12:07:33,551] INFO : Step 1100/50000 rank5 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1968 | Classification_binary: 0.4215 | Regression_cos_phi: 0.1683 | Regression_energy: 0.2290 | Regression_eta: 0.1688 | Regression_pt: 0.2356 | Regression_sin_phi: 0.1738 | LR: 9.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8135s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank5 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank5 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2234 | Classification_binary: 0.4507 | Regression_cos_phi: 0.2529 | Regression_energy: 0.2665 | Regression_eta: 0.1757 | Regression_pt: 0.2771 | Regression_sin_phi: 0.2478 | LR: 9.99e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4372s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank5 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank5 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1745 | Classification_binary: 0.3915 | Regression_cos_phi: 0.2112 | Regression_energy: 0.2239 | Regression_eta: 0.1502 | Regression_pt: 0.2284 | Regression_sin_phi: 0.2343 | LR: 9.98e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.2252s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank5 [2026-10-28 12:21:55,310] INFO : Step 1400/50000 rank5 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1891 | Classification_binary: 0.4032 | Regression_cos_phi: 0.2040 | Regression_energy: 0.2288 | Regression_eta: 0.1778 | Regression_pt: 0.2365 | Regression_sin_phi: 0.1976 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5558s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank5 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank5 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.4158 | Regression_cos_phi: 0.2217 | Regression_energy: 0.1991 | Regression_eta: 0.1685 | Regression_pt: 0.2078 | Regression_sin_phi: 0.2032 | LR: 9.98e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9970s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank5 [2026-10-28 12:31:13,938] INFO : Step 1600/50000 rank5 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2137 | Classification_binary: 0.4380 | Regression_cos_phi: 0.2783 | Regression_energy: 0.2243 | Regression_eta: 0.1974 | Regression_pt: 0.2351 | Regression_sin_phi: 0.2944 | LR: 9.98e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8061s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank5 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank5 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1435 | Classification_binary: 0.3528 | Regression_cos_phi: 0.2018 | Regression_energy: 0.2109 | Regression_eta: 0.1609 | Regression_pt: 0.2230 | Regression_sin_phi: 0.1662 | LR: 9.97e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank5 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank5 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2011 | Classification_binary: 0.3815 | Regression_cos_phi: 0.2055 | Regression_energy: 0.2029 | Regression_eta: 0.1735 | Regression_pt: 0.2169 | Regression_sin_phi: 0.1935 | LR: 9.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6767s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank5 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank5 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1712 | Classification_binary: 0.4303 | Regression_cos_phi: 0.2159 | Regression_energy: 0.2228 | Regression_eta: 0.1537 | Regression_pt: 0.2339 | Regression_sin_phi: 0.2131 | LR: 9.97e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4300s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank5 [2026-10-28 12:50:22,594] INFO : Step 2000/50000 rank5 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.3733 | Regression_cos_phi: 0.1837 | Regression_energy: 0.2249 | Regression_eta: 0.1573 | Regression_pt: 0.2304 | Regression_sin_phi: 0.1373 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9900s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank5 [2026-10-28 12:50:23,688] INFO : Running validation on rank5 +rank5 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 12:51:19,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:51:20,335] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 12:51:47,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:51:48,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:51:49,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:51:51,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:52:01,186] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 12:52:18,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:52:19,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:52:20,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:52:21,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:52:31,596] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 12:52:31,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:52:32,092] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 12:52:48,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:52:48,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:52:50,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:52:51,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:53:01,162] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank5 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1876 | Classification_binary: 0.3345 | Regression_cos_phi: 0.2261 | Regression_energy: 0.2521 | Regression_eta: 0.1439 | Regression_pt: 0.2596 | Regression_sin_phi: 0.1949 | LR: 9.96e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6446s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank5 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank5 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2090 | Classification_binary: 0.4251 | Regression_cos_phi: 0.2819 | Regression_energy: 0.2541 | Regression_eta: 0.1817 | Regression_pt: 0.2650 | Regression_sin_phi: 0.2587 | LR: 9.96e-04 | DataLoad Time: 0.0190s | Model Forward Time: 6.4357s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank5 [2026-10-28 13:08:12,324] INFO : Step 2300/50000 rank5 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1845 | Classification_binary: 0.3976 | Regression_cos_phi: 0.2217 | Regression_energy: 0.2313 | Regression_eta: 0.1962 | Regression_pt: 0.2413 | Regression_sin_phi: 0.2366 | LR: 9.95e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.1096s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank5 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank5 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1545 | Classification_binary: 0.3236 | Regression_cos_phi: 0.1979 | Regression_energy: 0.2614 | Regression_eta: 0.1459 | Regression_pt: 0.2662 | Regression_sin_phi: 0.1992 | LR: 9.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2197s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank5 [2026-10-28 13:18:06,890] INFO : Step 2500/50000 rank5 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2210 | Classification_binary: 0.4264 | Regression_cos_phi: 0.2485 | Regression_energy: 0.2270 | Regression_eta: 0.2072 | Regression_pt: 0.2351 | Regression_sin_phi: 0.2639 | LR: 9.94e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.0173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank5 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank5 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1944 | Classification_binary: 0.4459 | Regression_cos_phi: 0.2873 | Regression_energy: 0.2254 | Regression_eta: 0.1989 | Regression_pt: 0.2266 | Regression_sin_phi: 0.2620 | LR: 9.94e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.3801s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank5 [2026-10-28 13:27:39,806] INFO : Step 2700/50000 rank5 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1496 | Classification_binary: 0.3417 | Regression_cos_phi: 0.2151 | Regression_energy: 0.2033 | Regression_eta: 0.1936 | Regression_pt: 0.2038 | Regression_sin_phi: 0.2623 | LR: 9.94e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5213s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank5 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank5 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1609 | Classification_binary: 0.3463 | Regression_cos_phi: 0.1843 | Regression_energy: 0.2101 | Regression_eta: 0.1558 | Regression_pt: 0.2155 | Regression_sin_phi: 0.1848 | LR: 9.93e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2929s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank5 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank5 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1739 | Classification_binary: 0.4589 | Regression_cos_phi: 0.2375 | Regression_energy: 0.2132 | Regression_eta: 0.1907 | Regression_pt: 0.2212 | Regression_sin_phi: 0.2583 | LR: 9.93e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5107s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank5 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank5 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1576 | Classification_binary: 0.3670 | Regression_cos_phi: 0.2311 | Regression_energy: 0.2014 | Regression_eta: 0.1604 | Regression_pt: 0.2084 | Regression_sin_phi: 0.2495 | LR: 9.92e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5882s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank5 [2026-10-28 13:42:04,672] INFO : Running validation on rank5 +rank5 [2026-10-28 13:42:04,672] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:42:04,673] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:42:57,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:42:58,469] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 13:43:25,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:43:26,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:43:27,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:43:28,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:43:38,195] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:43:39,344] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 13:43:55,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:43:56,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:43:57,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:43:58,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:44:08,626] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:44:09,104] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 13:44:24,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:44:25,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:44:26,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:44:27,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:44:37,225] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 13:49:29,793] INFO : Step 3100/50000 rank5 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1498 | Classification_binary: 0.3156 | Regression_cos_phi: 0.1713 | Regression_energy: 0.1895 | Regression_eta: 0.1047 | Regression_pt: 0.1960 | Regression_sin_phi: 0.1448 | LR: 9.91e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8438s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank5 [2026-10-28 13:54:22,657] INFO : Step 3200/50000 rank5 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 0.3526 | Regression_cos_phi: 0.2039 | Regression_energy: 0.1909 | Regression_eta: 0.1296 | Regression_pt: 0.1965 | Regression_sin_phi: 0.1978 | LR: 9.91e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.0527s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank5 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank5 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1671 | Classification_binary: 0.3639 | Regression_cos_phi: 0.2335 | Regression_energy: 0.2146 | Regression_eta: 0.1710 | Regression_pt: 0.2269 | Regression_sin_phi: 0.2197 | LR: 9.90e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7403s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank5 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank5 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1158 | Classification_binary: 0.2738 | Regression_cos_phi: 0.1434 | Regression_energy: 0.2159 | Regression_eta: 0.1404 | Regression_pt: 0.2230 | Regression_sin_phi: 0.1378 | LR: 9.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9454s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank5 [2026-10-28 14:08:16,394] INFO : Step 3500/50000 rank5 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1848 | Classification_binary: 0.3711 | Regression_cos_phi: 0.2068 | Regression_energy: 0.1866 | Regression_eta: 0.1390 | Regression_pt: 0.1931 | Regression_sin_phi: 0.2044 | LR: 9.89e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7449s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank5 [2026-10-28 14:13:02,152] INFO : Step 3600/50000 rank5 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1337 | Classification_binary: 0.3840 | Regression_cos_phi: 0.1692 | Regression_energy: 0.2023 | Regression_eta: 0.1248 | Regression_pt: 0.2088 | Regression_sin_phi: 0.1798 | LR: 9.89e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8793s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank5 [2026-10-28 14:17:42,410] INFO : Step 3700/50000 rank5 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1345 | Classification_binary: 0.3129 | Regression_cos_phi: 0.1703 | Regression_energy: 0.1565 | Regression_eta: 0.1459 | Regression_pt: 0.1630 | Regression_sin_phi: 0.1901 | LR: 9.88e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2565s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank5 [2026-10-28 14:22:44,343] INFO : Step 3800/50000 rank5 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1658 | Classification_binary: 0.2974 | Regression_cos_phi: 0.1822 | Regression_energy: 0.2106 | Regression_eta: 0.1438 | Regression_pt: 0.2204 | Regression_sin_phi: 0.2217 | LR: 9.87e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.1813s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank5 [2026-10-28 14:27:33,634] INFO : Step 3900/50000 rank5 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1724 | Classification_binary: 0.4030 | Regression_cos_phi: 0.2409 | Regression_energy: 0.1959 | Regression_eta: 0.1480 | Regression_pt: 0.2061 | Regression_sin_phi: 0.1975 | LR: 9.87e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8174s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank5 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank5 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1304 | Classification_binary: 0.3878 | Regression_cos_phi: 0.2206 | Regression_energy: 0.1735 | Regression_eta: 0.1532 | Regression_pt: 0.1805 | Regression_sin_phi: 0.1995 | LR: 9.86e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5858s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank5 [2026-10-28 14:32:39,608] INFO : Running validation on rank5 +rank5 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 14:33:34,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:33:34,721] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 14:34:03,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 14:34:04,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 14:34:08,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 14:34:10,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 14:34:20,486] INFO : Time taken to make predictions on device 5 is: 0.33 min +rank5 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:34:24,284] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 14:34:40,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 14:34:43,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 14:34:44,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 14:34:45,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 14:34:55,285] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:34:56,346] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 14:35:11,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 14:35:12,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 14:35:13,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 14:35:14,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 14:35:24,452] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank5 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1159 | Classification_binary: 0.2776 | Regression_cos_phi: 0.1564 | Regression_energy: 0.1823 | Regression_eta: 0.1061 | Regression_pt: 0.1914 | Regression_sin_phi: 0.1408 | LR: 9.85e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8521s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank5 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank5 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1678 | Classification_binary: 0.3414 | Regression_cos_phi: 0.2111 | Regression_energy: 0.1993 | Regression_eta: 0.1398 | Regression_pt: 0.2072 | Regression_sin_phi: 0.2156 | LR: 9.84e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5093s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank5 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank5 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1584 | Classification_binary: 0.3551 | Regression_cos_phi: 0.2033 | Regression_energy: 0.2200 | Regression_eta: 0.1400 | Regression_pt: 0.2291 | Regression_sin_phi: 0.1731 | LR: 9.84e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4594s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank5 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank5 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1510 | Regression_energy: 0.1890 | Regression_eta: 0.1270 | Regression_pt: 0.1912 | Regression_sin_phi: 0.1536 | LR: 9.83e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1729s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank5 [2026-10-28 14:59:28,687] INFO : Step 4500/50000 rank5 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1399 | Classification_binary: 0.3162 | Regression_cos_phi: 0.1562 | Regression_energy: 0.1785 | Regression_eta: 0.1327 | Regression_pt: 0.1874 | Regression_sin_phi: 0.2209 | LR: 9.82e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2513s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank5 [2026-10-28 15:04:24,812] INFO : Step 4600/50000 rank5 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1397 | Classification_binary: 0.4007 | Regression_cos_phi: 0.2292 | Regression_energy: 0.1967 | Regression_eta: 0.1495 | Regression_pt: 0.2053 | Regression_sin_phi: 0.2041 | LR: 9.81e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7026s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank5 [2026-10-28 15:09:18,560] INFO : Step 4700/50000 rank5 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1367 | Classification_binary: 0.3583 | Regression_cos_phi: 0.1804 | Regression_energy: 0.1885 | Regression_eta: 0.1589 | Regression_pt: 0.1946 | Regression_sin_phi: 0.2428 | LR: 9.81e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7060s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank5 [2026-10-28 15:14:30,753] INFO : Step 4800/50000 rank5 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1302 | Classification_binary: 0.3004 | Regression_cos_phi: 0.2213 | Regression_energy: 0.1679 | Regression_eta: 0.1063 | Regression_pt: 0.1738 | Regression_sin_phi: 0.1696 | LR: 9.80e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4406s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank5 [2026-10-28 15:19:27,570] INFO : Step 4900/50000 rank5 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.3862 | Regression_cos_phi: 0.2334 | Regression_energy: 0.1751 | Regression_eta: 0.1961 | Regression_pt: 0.1827 | Regression_sin_phi: 0.2670 | LR: 9.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5348s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank5 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank5 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1099 | Classification_binary: 0.3479 | Regression_cos_phi: 0.1962 | Regression_energy: 0.1638 | Regression_eta: 0.1206 | Regression_pt: 0.1670 | Regression_sin_phi: 0.1753 | LR: 9.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4823s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank5 [2026-10-28 15:24:24,982] INFO : Running validation on rank5 +rank5 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 15:25:19,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:25:19,570] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 15:25:45,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 15:25:46,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 15:25:47,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 15:25:48,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 15:25:58,033] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:25:59,251] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 15:26:15,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 15:26:16,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 15:26:17,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 15:26:18,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 15:26:28,180] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:26:28,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 15:26:44,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 15:26:45,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 15:26:46,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 15:26:47,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 15:26:57,544] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 15:32:01,930] INFO : Step 5100/50000 rank5 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2701 | Regression_cos_phi: 0.1371 | Regression_energy: 0.1897 | Regression_eta: 0.1215 | Regression_pt: 0.1945 | Regression_sin_phi: 0.1438 | LR: 9.77e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.6418s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank5 [2026-10-28 15:36:53,511] INFO : Step 5200/50000 rank5 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1573 | Classification_binary: 0.3555 | Regression_cos_phi: 0.1796 | Regression_energy: 0.1836 | Regression_eta: 0.1314 | Regression_pt: 0.1909 | Regression_sin_phi: 0.1990 | LR: 9.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6371s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank5 [2026-10-28 15:41:36,525] INFO : Step 5300/50000 rank5 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3309 | Regression_cos_phi: 0.1698 | Regression_energy: 0.1709 | Regression_eta: 0.1351 | Regression_pt: 0.1744 | Regression_sin_phi: 0.1759 | LR: 9.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9244s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank5 [2026-10-28 15:46:33,435] INFO : Step 5400/50000 rank5 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1074 | Classification_binary: 0.3392 | Regression_cos_phi: 0.1630 | Regression_energy: 0.1405 | Regression_eta: 0.1304 | Regression_pt: 0.1492 | Regression_sin_phi: 0.1548 | LR: 9.74e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4636s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank5 [2026-10-28 15:51:26,878] INFO : Step 5500/50000 rank5 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.3126 | Regression_cos_phi: 0.1729 | Regression_energy: 0.1636 | Regression_eta: 0.1064 | Regression_pt: 0.1720 | Regression_sin_phi: 0.1850 | LR: 9.73e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.5342s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank5 [2026-10-28 15:56:25,711] INFO : Step 5600/50000 rank5 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 0.4381 | Regression_cos_phi: 0.1747 | Regression_energy: 0.1800 | Regression_eta: 0.1676 | Regression_pt: 0.1883 | Regression_sin_phi: 0.1930 | LR: 9.72e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.3182s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank5 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank5 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1117 | Classification_binary: 0.3001 | Regression_cos_phi: 0.1520 | Regression_energy: 0.1578 | Regression_eta: 0.1214 | Regression_pt: 0.1650 | Regression_sin_phi: 0.1842 | LR: 9.71e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.5504s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank5 [2026-10-28 16:06:09,458] INFO : Step 5800/50000 rank5 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1512 | Classification_binary: 0.2945 | Regression_cos_phi: 0.2197 | Regression_energy: 0.1883 | Regression_eta: 0.1594 | Regression_pt: 0.1938 | Regression_sin_phi: 0.2554 | LR: 9.70e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2267s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank5 [2026-10-28 16:11:19,901] INFO : Step 5900/50000 rank5 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1579 | Classification_binary: 0.3936 | Regression_cos_phi: 0.2262 | Regression_energy: 0.1841 | Regression_eta: 0.1591 | Regression_pt: 0.1948 | Regression_sin_phi: 0.1971 | LR: 9.69e-04 | DataLoad Time: 0.0160s | Model Forward Time: 7.1479s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank5 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank5 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1451 | Classification_binary: 0.3634 | Regression_cos_phi: 0.2259 | Regression_energy: 0.1684 | Regression_eta: 0.1380 | Regression_pt: 0.1750 | Regression_sin_phi: 0.1741 | LR: 9.68e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4962s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank5 [2026-10-28 16:16:41,624] INFO : Running validation on rank5 +rank5 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:17:32,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:17:33,086] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 16:17:59,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:18:00,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:18:01,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:18:02,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:18:12,677] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:18:13,874] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 16:18:29,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:18:30,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:18:31,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:18:32,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:18:42,645] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:18:43,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:18:43,951] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 16:18:59,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:19:00,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:19:02,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:19:04,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:19:14,125] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank5 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1039 | Classification_binary: 0.2197 | Regression_cos_phi: 0.1574 | Regression_energy: 0.1740 | Regression_eta: 0.1027 | Regression_pt: 0.1811 | Regression_sin_phi: 0.1033 | LR: 9.67e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9556s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank5 [2026-10-28 16:29:26,888] INFO : Step 6200/50000 rank5 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1326 | Classification_binary: 0.2648 | Regression_cos_phi: 0.1572 | Regression_energy: 0.1657 | Regression_eta: 0.1132 | Regression_pt: 0.1756 | Regression_sin_phi: 0.1457 | LR: 9.66e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0888s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank5 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank5 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1249 | Classification_binary: 0.3208 | Regression_cos_phi: 0.1790 | Regression_energy: 0.1696 | Regression_eta: 0.1219 | Regression_pt: 0.1761 | Regression_sin_phi: 0.1513 | LR: 9.65e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.0298s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank5 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank5 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0965 | Classification_binary: 0.2930 | Regression_cos_phi: 0.1746 | Regression_energy: 0.1972 | Regression_eta: 0.1024 | Regression_pt: 0.2068 | Regression_sin_phi: 0.1405 | LR: 9.64e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7772s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank5 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank5 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.2647 | Regression_cos_phi: 0.1507 | Regression_energy: 0.1495 | Regression_eta: 0.0820 | Regression_pt: 0.1558 | Regression_sin_phi: 0.1435 | LR: 9.63e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.4995s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank5 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank5 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1352 | Classification_binary: 0.3550 | Regression_cos_phi: 0.1761 | Regression_energy: 0.1759 | Regression_eta: 0.1311 | Regression_pt: 0.1842 | Regression_sin_phi: 0.1655 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8071s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank5 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank5 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1372 | Classification_binary: 0.3286 | Regression_cos_phi: 0.1992 | Regression_energy: 0.1439 | Regression_eta: 0.1549 | Regression_pt: 0.1473 | Regression_sin_phi: 0.1709 | LR: 9.61e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3372s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank5 [2026-10-28 16:58:34,482] INFO : Step 6800/50000 rank5 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1688 | Classification_binary: 0.2724 | Regression_cos_phi: 0.1919 | Regression_energy: 0.1594 | Regression_eta: 0.1268 | Regression_pt: 0.1691 | Regression_sin_phi: 0.2008 | LR: 9.60e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.4638s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank5 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank5 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1403 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1947 | Regression_energy: 0.1613 | Regression_eta: 0.1150 | Regression_pt: 0.1661 | Regression_sin_phi: 0.1914 | LR: 9.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6878s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank5 [2026-10-28 17:08:29,915] INFO : Step 7000/50000 rank5 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1384 | Classification_binary: 0.3410 | Regression_cos_phi: 0.1567 | Regression_energy: 0.1549 | Regression_eta: 0.1214 | Regression_pt: 0.1580 | Regression_sin_phi: 0.1754 | LR: 9.57e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.9214s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank5 [2026-10-28 17:08:30,791] INFO : Running validation on rank5 +rank5 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 17:09:23,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:09:24,267] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 17:09:52,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 17:09:53,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 17:09:55,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 17:09:56,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 17:10:06,733] INFO : Time taken to make predictions on device 5 is: 0.29 min +rank5 [2026-10-28 17:10:07,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:10:08,261] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 17:10:24,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 17:10:25,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 17:10:26,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 17:10:27,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 17:10:37,359] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 17:10:37,866] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:10:38,448] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 17:10:54,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 17:10:55,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 17:10:56,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 17:10:57,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 17:11:07,100] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 17:16:27,595] INFO : Step 7100/50000 rank5 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2740 | Regression_cos_phi: 0.1123 | Regression_energy: 0.1462 | Regression_eta: 0.0944 | Regression_pt: 0.1511 | Regression_sin_phi: 0.1145 | LR: 9.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1470s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank5 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank5 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1408 | Classification_binary: 0.3700 | Regression_cos_phi: 0.2024 | Regression_energy: 0.1446 | Regression_eta: 0.1384 | Regression_pt: 0.1500 | Regression_sin_phi: 0.2233 | LR: 9.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3460s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank5 [2026-10-28 17:26:34,943] INFO : Step 7300/50000 rank5 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1446 | Classification_binary: 0.3726 | Regression_cos_phi: 0.2184 | Regression_energy: 0.1498 | Regression_eta: 0.1316 | Regression_pt: 0.1591 | Regression_sin_phi: 0.1797 | LR: 9.53e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4021s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank5 [2026-10-28 17:31:34,683] INFO : Step 7400/50000 rank5 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2495 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1477 | Regression_eta: 0.0815 | Regression_pt: 0.1519 | Regression_sin_phi: 0.1303 | LR: 9.52e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank5 [2026-10-28 17:36:30,885] INFO : Step 7500/50000 rank5 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1082 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1419 | Regression_energy: 0.1647 | Regression_eta: 0.1094 | Regression_pt: 0.1725 | Regression_sin_phi: 0.1705 | LR: 9.51e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6607s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank5 [2026-10-28 17:41:09,113] INFO : Step 7600/50000 rank5 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1235 | Classification_binary: 0.3378 | Regression_cos_phi: 0.1588 | Regression_energy: 0.1509 | Regression_eta: 0.1086 | Regression_pt: 0.1575 | Regression_sin_phi: 0.1418 | LR: 9.50e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7908s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank5 [2026-10-28 17:46:03,243] INFO : Step 7700/50000 rank5 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.2766 | Regression_cos_phi: 0.1320 | Regression_energy: 0.1346 | Regression_eta: 0.0882 | Regression_pt: 0.1367 | Regression_sin_phi: 0.1417 | LR: 9.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7818s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank5 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank5 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1291 | Classification_binary: 0.2484 | Regression_cos_phi: 0.1491 | Regression_energy: 0.1380 | Regression_eta: 0.1140 | Regression_pt: 0.1446 | Regression_sin_phi: 0.1513 | LR: 9.47e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4023s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank5 [2026-10-28 17:56:05,209] INFO : Step 7900/50000 rank5 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1565 | Classification_binary: 0.3413 | Regression_cos_phi: 0.1594 | Regression_energy: 0.1468 | Regression_eta: 0.1396 | Regression_pt: 0.1517 | Regression_sin_phi: 0.1950 | LR: 9.46e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.3975s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank5 [2026-10-28 18:00:55,277] INFO : Step 8000/50000 rank5 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1238 | Classification_binary: 0.3077 | Regression_cos_phi: 0.1455 | Regression_energy: 0.1438 | Regression_eta: 0.1185 | Regression_pt: 0.1490 | Regression_sin_phi: 0.1315 | LR: 9.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4815s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank5 [2026-10-28 18:00:56,228] INFO : Running validation on rank5 +rank5 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:01:47,759] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:01:48,315] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 18:02:14,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:02:15,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:02:16,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:02:17,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:02:27,923] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:02:28,857] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 18:02:44,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:02:45,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:02:46,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:02:47,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:02:57,310] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:02:58,281] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:02:58,745] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 18:03:14,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:03:15,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:03:15,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:03:16,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:03:26,869] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 18:08:30,022] INFO : Step 8100/50000 rank5 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2201 | Regression_cos_phi: 0.1087 | Regression_energy: 0.1476 | Regression_eta: 0.0837 | Regression_pt: 0.1489 | Regression_sin_phi: 0.1159 | LR: 9.43e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.3410s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank5 [2026-10-28 18:13:41,921] INFO : Step 8200/50000 rank5 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1132 | Classification_binary: 0.3136 | Regression_cos_phi: 0.1126 | Regression_energy: 0.1380 | Regression_eta: 0.1046 | Regression_pt: 0.1422 | Regression_sin_phi: 0.1594 | LR: 9.42e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3267s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank5 [2026-10-28 18:18:39,531] INFO : Step 8300/50000 rank5 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1023 | Classification_binary: 0.3308 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1323 | Regression_eta: 0.1204 | Regression_pt: 0.1367 | Regression_sin_phi: 0.1564 | LR: 9.40e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.6361s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank5 [2026-10-28 18:23:43,087] INFO : Step 8400/50000 rank5 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2393 | Regression_cos_phi: 0.1188 | Regression_energy: 0.1336 | Regression_eta: 0.1016 | Regression_pt: 0.1355 | Regression_sin_phi: 0.1387 | LR: 9.39e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.1826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank5 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank5 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2918 | Regression_cos_phi: 0.1559 | Regression_energy: 0.1385 | Regression_eta: 0.1245 | Regression_pt: 0.1430 | Regression_sin_phi: 0.1381 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5043s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank5 [2026-10-28 18:34:09,473] INFO : Step 8600/50000 rank5 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0933 | Classification_binary: 0.3465 | Regression_cos_phi: 0.2031 | Regression_energy: 0.1402 | Regression_eta: 0.1396 | Regression_pt: 0.1457 | Regression_sin_phi: 0.1678 | LR: 9.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7743s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank5 [2026-10-28 18:39:14,578] INFO : Step 8700/50000 rank5 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0924 | Classification_binary: 0.2607 | Regression_cos_phi: 0.1215 | Regression_energy: 0.1287 | Regression_eta: 0.0887 | Regression_pt: 0.1332 | Regression_sin_phi: 0.1437 | LR: 9.34e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7200s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank5 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank5 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2409 | Regression_cos_phi: 0.1565 | Regression_energy: 0.1339 | Regression_eta: 0.0966 | Regression_pt: 0.1361 | Regression_sin_phi: 0.1711 | LR: 9.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.4104s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank5 [2026-10-28 18:49:28,516] INFO : Step 8900/50000 rank5 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0970 | Classification_binary: 0.3094 | Regression_cos_phi: 0.1391 | Regression_energy: 0.1413 | Regression_eta: 0.1266 | Regression_pt: 0.1471 | Regression_sin_phi: 0.1264 | LR: 9.31e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7918s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank5 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank5 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0896 | Classification_binary: 0.3114 | Regression_cos_phi: 0.1291 | Regression_energy: 0.1336 | Regression_eta: 0.0905 | Regression_pt: 0.1373 | Regression_sin_phi: 0.1278 | LR: 9.30e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8395s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank5 [2026-10-28 18:54:54,936] INFO : Running validation on rank5 +rank5 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:55:52,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:55:53,257] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 18:56:19,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:56:21,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:56:22,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:56:23,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:56:34,006] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:56:35,475] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 18:56:51,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:56:52,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:56:53,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:56:54,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:57:04,829] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:57:05,415] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:57:05,961] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 18:57:21,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:57:22,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:57:22,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:57:23,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:57:33,809] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 19:03:08,598] INFO : Step 9100/50000 rank5 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2205 | Regression_cos_phi: 0.1088 | Regression_energy: 0.1354 | Regression_eta: 0.0644 | Regression_pt: 0.1404 | Regression_sin_phi: 0.1061 | LR: 9.28e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.1689s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank5 [2026-10-28 19:08:18,223] INFO : Step 9200/50000 rank5 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1072 | Classification_binary: 0.2715 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1316 | Regression_eta: 0.1023 | Regression_pt: 0.1373 | Regression_sin_phi: 0.1788 | LR: 9.27e-04 | DataLoad Time: 0.0251s | Model Forward Time: 6.5440s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank5 [2026-10-28 19:13:30,253] INFO : Step 9300/50000 rank5 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1317 | Classification_binary: 0.3294 | Regression_cos_phi: 0.1579 | Regression_energy: 0.1396 | Regression_eta: 0.1029 | Regression_pt: 0.1482 | Regression_sin_phi: 0.1646 | LR: 9.25e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3913s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank5 [2026-10-28 19:18:55,622] INFO : Step 9400/50000 rank5 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1429 | Regression_energy: 0.1361 | Regression_eta: 0.1154 | Regression_pt: 0.1349 | Regression_sin_phi: 0.1748 | LR: 9.24e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2017s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank5 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank5 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2158 | Regression_cos_phi: 0.1106 | Regression_energy: 0.1254 | Regression_eta: 0.0808 | Regression_pt: 0.1281 | Regression_sin_phi: 0.1046 | LR: 9.22e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2843s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank5 [2026-10-28 19:29:34,439] INFO : Step 9600/50000 rank5 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0900 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1524 | Regression_energy: 0.1403 | Regression_eta: 0.0892 | Regression_pt: 0.1450 | Regression_sin_phi: 0.1610 | LR: 9.21e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6040s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank5 [2026-10-28 19:34:53,790] INFO : Step 9700/50000 rank5 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2684 | Regression_cos_phi: 0.1143 | Regression_energy: 0.1176 | Regression_eta: 0.0788 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0850 | LR: 9.19e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9751s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank5 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank5 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0818 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0882 | Regression_energy: 0.1482 | Regression_eta: 0.0705 | Regression_pt: 0.1539 | Regression_sin_phi: 0.0980 | LR: 9.17e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5617s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank5 [2026-10-28 19:44:46,547] INFO : Step 9900/50000 rank5 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1065 | Classification_binary: 0.3095 | Regression_cos_phi: 0.1981 | Regression_energy: 0.1349 | Regression_eta: 0.1123 | Regression_pt: 0.1432 | Regression_sin_phi: 0.1698 | LR: 9.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9308s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank5 [2026-10-28 19:49:37,630] INFO : Step 10000/50000 rank5 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.2863 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1271 | Regression_eta: 0.0950 | Regression_pt: 0.1320 | Regression_sin_phi: 0.1246 | LR: 9.14e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.6251s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank5 [2026-10-28 19:49:38,691] INFO : Running validation on rank5 +rank5 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 19:50:35,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:50:35,624] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 19:51:01,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 19:51:02,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 19:51:03,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 19:51:04,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 19:51:14,857] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:51:15,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:51:16,383] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 19:51:32,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 19:51:33,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 19:51:34,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 19:51:35,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 19:51:45,390] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:51:46,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:51:46,687] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 19:52:02,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 19:52:02,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 19:52:03,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 19:52:04,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 19:52:14,707] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 19:57:27,968] INFO : Step 10100/50000 rank5 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2624 | Regression_cos_phi: 0.1250 | Regression_energy: 0.1277 | Regression_eta: 0.0787 | Regression_pt: 0.1302 | Regression_sin_phi: 0.1191 | LR: 9.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7465s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank5 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank5 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2666 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1096 | Regression_eta: 0.0782 | Regression_pt: 0.1158 | Regression_sin_phi: 0.1028 | LR: 9.11e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6810s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank5 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank5 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.3182 | Regression_cos_phi: 0.1223 | Regression_energy: 0.1238 | Regression_eta: 0.0899 | Regression_pt: 0.1263 | Regression_sin_phi: 0.1358 | LR: 9.09e-04 | DataLoad Time: 0.0227s | Model Forward Time: 5.1994s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank5 [2026-10-28 20:12:49,148] INFO : Step 10400/50000 rank5 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0939 | Classification_binary: 0.2227 | Regression_cos_phi: 0.1645 | Regression_energy: 0.1161 | Regression_eta: 0.1001 | Regression_pt: 0.1236 | Regression_sin_phi: 0.1513 | LR: 9.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2218s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank5 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank5 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2342 | Regression_cos_phi: 0.1141 | Regression_energy: 0.1158 | Regression_eta: 0.0561 | Regression_pt: 0.1192 | Regression_sin_phi: 0.1021 | LR: 9.06e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3239s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank5 [2026-10-28 20:23:17,056] INFO : Step 10600/50000 rank5 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2688 | Regression_cos_phi: 0.1277 | Regression_energy: 0.1092 | Regression_eta: 0.0865 | Regression_pt: 0.1138 | Regression_sin_phi: 0.1232 | LR: 9.04e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0933s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank5 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank5 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2641 | Regression_cos_phi: 0.1165 | Regression_energy: 0.1149 | Regression_eta: 0.0738 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1069 | LR: 9.02e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8724s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank5 [2026-10-28 20:33:28,995] INFO : Step 10800/50000 rank5 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1039 | Regression_eta: 0.0496 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0768 | LR: 9.00e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6831s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank5 [2026-10-28 20:38:44,061] INFO : Step 10900/50000 rank5 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2360 | Regression_cos_phi: 0.1121 | Regression_energy: 0.1139 | Regression_eta: 0.0665 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0923 | LR: 8.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7058s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank5 [2026-10-28 20:43:46,527] INFO : Step 11000/50000 rank5 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.3031 | Regression_cos_phi: 0.1248 | Regression_energy: 0.1072 | Regression_eta: 0.0897 | Regression_pt: 0.1122 | Regression_sin_phi: 0.1184 | LR: 8.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4437s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank5 [2026-10-28 20:43:47,416] INFO : Running validation on rank5 +rank5 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 20:44:41,298] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 20:45:10,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 20:45:11,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 20:45:12,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 20:45:13,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 20:45:23,116] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:45:23,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:45:24,126] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 20:45:40,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 20:45:41,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 20:45:43,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 20:45:44,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 20:45:54,286] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 20:46:11,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 20:46:12,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 20:46:13,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 20:46:15,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 20:46:25,029] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:51:55,949] INFO : Step 11100/50000 rank5 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0779 | Regression_energy: 0.1081 | Regression_eta: 0.0638 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0684 | LR: 8.95e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8315s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank5 [2026-10-28 20:57:10,090] INFO : Step 11200/50000 rank5 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.1761 | Regression_cos_phi: 0.1063 | Regression_energy: 0.1055 | Regression_eta: 0.0687 | Regression_pt: 0.1081 | Regression_sin_phi: 0.1216 | LR: 8.93e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8529s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank5 [2026-10-28 21:02:18,460] INFO : Step 11300/50000 rank5 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2994 | Regression_cos_phi: 0.1453 | Regression_energy: 0.1037 | Regression_eta: 0.0918 | Regression_pt: 0.1097 | Regression_sin_phi: 0.1341 | LR: 8.91e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2912s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank5 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank5 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2346 | Regression_cos_phi: 0.1197 | Regression_energy: 0.0994 | Regression_eta: 0.0676 | Regression_pt: 0.1021 | Regression_sin_phi: 0.1076 | LR: 8.89e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9837s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank5 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank5 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0828 | Regression_energy: 0.1163 | Regression_eta: 0.0629 | Regression_pt: 0.1197 | Regression_sin_phi: 0.0703 | LR: 8.88e-04 | DataLoad Time: 0.0129s | Model Forward Time: 4.3459s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank5 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank5 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2447 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0962 | Regression_eta: 0.0589 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0834 | LR: 8.86e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2597s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank5 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank5 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0962 | Classification_binary: 0.2758 | Regression_cos_phi: 0.1276 | Regression_energy: 0.1051 | Regression_eta: 0.0732 | Regression_pt: 0.1086 | Regression_sin_phi: 0.1290 | LR: 8.84e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank5 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank5 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0933 | Regression_energy: 0.1069 | Regression_eta: 0.0536 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0652 | LR: 8.82e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5590s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank5 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank5 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2320 | Regression_cos_phi: 0.1353 | Regression_energy: 0.1128 | Regression_eta: 0.0806 | Regression_pt: 0.1154 | Regression_sin_phi: 0.1039 | LR: 8.80e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9849s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank5 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank5 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2871 | Regression_cos_phi: 0.0998 | Regression_energy: 0.1074 | Regression_eta: 0.0863 | Regression_pt: 0.1128 | Regression_sin_phi: 0.1169 | LR: 8.78e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0206s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank5 [2026-10-28 21:36:38,362] INFO : Running validation on rank5 +rank5 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:37:31,336] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 21:38:00,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:38:01,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:38:02,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:38:03,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:38:13,502] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:38:14,610] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 21:38:30,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:38:32,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:38:34,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:38:35,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:38:45,154] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 21:38:45,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:38:46,120] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 21:39:01,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:39:02,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:39:03,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:39:04,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:39:14,190] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank5 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0783 | Regression_eta: 0.0592 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0890 | LR: 8.76e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.1525s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank5 [2026-10-28 21:49:00,968] INFO : Step 12200/50000 rank5 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2236 | Regression_cos_phi: 0.1045 | Regression_energy: 0.1020 | Regression_eta: 0.0787 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0934 | LR: 8.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5151s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank5 [2026-10-28 21:53:55,879] INFO : Step 12300/50000 rank5 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2666 | Regression_cos_phi: 0.1308 | Regression_energy: 0.1030 | Regression_eta: 0.0678 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1119 | LR: 8.72e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6878s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank5 [2026-10-28 21:58:40,947] INFO : Step 12400/50000 rank5 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2725 | Regression_cos_phi: 0.1097 | Regression_energy: 0.1131 | Regression_eta: 0.0855 | Regression_pt: 0.1163 | Regression_sin_phi: 0.1082 | LR: 8.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2689s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank5 [2026-10-28 22:03:23,385] INFO : Step 12500/50000 rank5 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1626 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0739 | Regression_eta: 0.0525 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0546 | LR: 8.68e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6714s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank5 [2026-10-28 22:08:04,813] INFO : Step 12600/50000 rank5 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2461 | Regression_cos_phi: 0.1098 | Regression_energy: 0.1100 | Regression_eta: 0.1007 | Regression_pt: 0.1146 | Regression_sin_phi: 0.1144 | LR: 8.66e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.1052s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank5 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank5 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2633 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0930 | Regression_eta: 0.0636 | Regression_pt: 0.0947 | Regression_sin_phi: 0.1086 | LR: 8.64e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3623s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank5 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank5 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0910 | Regression_eta: 0.0681 | Regression_pt: 0.0960 | Regression_sin_phi: 0.1045 | LR: 8.62e-04 | DataLoad Time: 0.0218s | Model Forward Time: 6.0978s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank5 [2026-10-28 22:22:32,703] INFO : Step 12900/50000 rank5 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2285 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0881 | Regression_eta: 0.0742 | Regression_pt: 0.0910 | Regression_sin_phi: 0.1102 | LR: 8.60e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6619s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank5 [2026-10-28 22:27:21,815] INFO : Step 13000/50000 rank5 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2391 | Regression_cos_phi: 0.1609 | Regression_energy: 0.0899 | Regression_eta: 0.0877 | Regression_pt: 0.0935 | Regression_sin_phi: 0.1085 | LR: 8.58e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.9255s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank5 [2026-10-28 22:27:23,210] INFO : Running validation on rank5 +rank5 [2026-10-28 22:27:23,210] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:28:28,170] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 22:28:54,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 22:28:55,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 22:28:56,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 22:28:57,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 22:29:07,115] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 22:29:08,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:29:08,604] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 22:29:24,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 22:29:25,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 22:29:26,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 22:29:27,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 22:29:37,665] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:29:38,493] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 22:29:53,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 22:29:54,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 22:29:55,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 22:29:56,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 22:30:06,538] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 22:35:15,707] INFO : Step 13100/50000 rank5 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0899 | Regression_eta: 0.0539 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0727 | LR: 8.56e-04 | DataLoad Time: 0.0225s | Model Forward Time: 6.4831s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank5 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank5 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2355 | Regression_cos_phi: 0.1058 | Regression_energy: 0.0959 | Regression_eta: 0.0952 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1194 | LR: 8.54e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6866s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank5 [2026-10-28 22:44:58,538] INFO : Step 13300/50000 rank5 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2433 | Regression_cos_phi: 0.1078 | Regression_energy: 0.0833 | Regression_eta: 0.0704 | Regression_pt: 0.0858 | Regression_sin_phi: 0.1281 | LR: 8.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6743s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank5 [2026-10-28 22:49:50,191] INFO : Step 13400/50000 rank5 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 0.3469 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1306 | Regression_eta: 0.0865 | Regression_pt: 0.1342 | Regression_sin_phi: 0.1132 | LR: 8.50e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.6014s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank5 [2026-10-28 22:54:39,510] INFO : Step 13500/50000 rank5 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0915 | Regression_energy: 0.1673 | Regression_eta: 0.0584 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0987 | LR: 8.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4618s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank5 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank5 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0896 | Regression_eta: 0.0546 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0720 | LR: 8.45e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8492s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank5 [2026-10-28 23:04:05,988] INFO : Step 13700/50000 rank5 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2766 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0862 | Regression_eta: 0.0477 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0674 | LR: 8.43e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7316s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank5 [2026-10-28 23:09:16,293] INFO : Step 13800/50000 rank5 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0730 | Regression_energy: 0.1102 | Regression_eta: 0.0535 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0851 | LR: 8.41e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4781s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank5 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank5 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1144 | Classification_binary: 0.2517 | Regression_cos_phi: 0.1326 | Regression_energy: 0.1161 | Regression_eta: 0.0798 | Regression_pt: 0.1195 | Regression_sin_phi: 0.1221 | LR: 8.39e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank5 [2026-10-28 23:18:59,722] INFO : Step 14000/50000 rank5 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.2866 | Regression_cos_phi: 0.1195 | Regression_energy: 0.0969 | Regression_eta: 0.0832 | Regression_pt: 0.0992 | Regression_sin_phi: 0.1093 | LR: 8.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1767s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank5 [2026-10-28 23:19:00,860] INFO : Running validation on rank5 +rank5 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:19:53,516] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-28 23:20:19,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 23:20:20,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 23:20:21,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 23:20:22,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 23:20:32,554] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:20:33,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:20:33,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-28 23:20:49,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 23:20:50,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 23:20:51,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 23:20:52,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 23:21:02,589] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:21:03,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:21:04,039] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-28 23:21:19,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 23:21:20,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 23:21:21,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 23:21:22,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 23:21:32,082] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 23:26:43,451] INFO : Step 14100/50000 rank5 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2467 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0871 | Regression_eta: 0.0809 | Regression_pt: 0.0906 | Regression_sin_phi: 0.1017 | LR: 8.35e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5308s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank5 [2026-10-28 23:31:31,720] INFO : Step 14200/50000 rank5 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0855 | Regression_eta: 0.0477 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0732 | LR: 8.32e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6170s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank5 [2026-10-28 23:36:16,487] INFO : Step 14300/50000 rank5 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2550 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0934 | Regression_eta: 0.0561 | Regression_pt: 0.0974 | Regression_sin_phi: 0.1178 | LR: 8.30e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6733s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank5 [2026-10-28 23:40:58,337] INFO : Step 14400/50000 rank5 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2815 | Regression_cos_phi: 0.1051 | Regression_energy: 0.0973 | Regression_eta: 0.0850 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0812 | LR: 8.28e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7759s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank5 [2026-10-28 23:45:42,941] INFO : Step 14500/50000 rank5 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0795 | Regression_eta: 0.0483 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0673 | LR: 8.26e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.6358s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank5 [2026-10-28 23:50:27,494] INFO : Step 14600/50000 rank5 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0840 | Regression_energy: 0.0828 | Regression_eta: 0.0680 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0930 | LR: 8.24e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank5 [2026-10-28 23:55:12,844] INFO : Step 14700/50000 rank5 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.2319 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0808 | Regression_eta: 0.0751 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0726 | LR: 8.21e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6462s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank5 [2026-10-28 23:59:56,486] INFO : Step 14800/50000 rank5 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0690 | Regression_eta: 0.0553 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0827 | LR: 8.19e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.7602s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank5 [2026-10-29 00:04:46,438] INFO : Step 14900/50000 rank5 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0774 | Regression_eta: 0.0468 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0650 | LR: 8.17e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1958s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank5 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank5 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2173 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0868 | Regression_eta: 0.0581 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0820 | LR: 8.15e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7489s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank5 [2026-10-29 00:09:44,171] INFO : Running validation on rank5 +rank5 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:10:38,762] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:10:39,638] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 00:11:05,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:11:07,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:11:08,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:11:09,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:11:19,329] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 00:11:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:11:20,772] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 00:11:36,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:11:37,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:11:38,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:11:39,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:11:49,606] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:11:50,988] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 00:12:06,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:12:07,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:12:08,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:12:09,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:12:19,322] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 00:17:30,255] INFO : Step 15100/50000 rank5 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0827 | Regression_eta: 0.0722 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0804 | LR: 8.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.3991s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank5 [2026-10-29 00:22:16,917] INFO : Step 15200/50000 rank5 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0638 | Regression_energy: 0.1037 | Regression_eta: 0.0460 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0738 | LR: 8.10e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2240s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank5 [2026-10-29 00:27:05,719] INFO : Step 15300/50000 rank5 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2186 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0872 | Regression_eta: 0.0723 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0817 | LR: 8.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3155s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank5 [2026-10-29 00:31:57,726] INFO : Step 15400/50000 rank5 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2421 | Regression_cos_phi: 0.1059 | Regression_energy: 0.0840 | Regression_eta: 0.0610 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0914 | LR: 8.05e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6644s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank5 [2026-10-29 00:36:52,790] INFO : Step 15500/50000 rank5 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0746 | Regression_eta: 0.0517 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0533 | LR: 8.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4224s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank5 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank5 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1884 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0702 | Regression_eta: 0.0749 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0878 | LR: 8.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9294s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank5 [2026-10-29 00:46:43,873] INFO : Step 15700/50000 rank5 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2689 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0838 | Regression_eta: 0.0535 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0991 | LR: 7.98e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7566s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank5 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank5 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.2383 | Regression_cos_phi: 0.1044 | Regression_energy: 0.0845 | Regression_eta: 0.0762 | Regression_pt: 0.0886 | Regression_sin_phi: 0.1032 | LR: 7.96e-04 | DataLoad Time: 0.0243s | Model Forward Time: 5.0241s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank5 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank5 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1912 | Regression_cos_phi: 0.1203 | Regression_energy: 0.0738 | Regression_eta: 0.0754 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0834 | LR: 7.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank5 [2026-10-29 01:01:16,824] INFO : Step 16000/50000 rank5 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2151 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0882 | Regression_eta: 0.0599 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0753 | LR: 7.91e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.3010s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank5 [2026-10-29 01:01:17,883] INFO : Running validation on rank5 +rank5 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 01:02:10,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:02:11,393] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 01:02:39,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 01:02:40,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 01:02:42,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 01:02:45,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 01:02:55,102] INFO : Time taken to make predictions on device 5 is: 0.30 min +rank5 [2026-10-29 01:02:55,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:02:55,637] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 01:03:11,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 01:03:12,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 01:03:13,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 01:03:15,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 01:03:25,066] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 01:03:25,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:03:26,088] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 01:03:42,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 01:03:42,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 01:03:43,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 01:03:44,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 01:03:54,617] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank5 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1139 | Regression_energy: 0.0858 | Regression_eta: 0.0707 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0931 | LR: 7.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0649s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank5 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank5 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1899 | Regression_cos_phi: 0.1046 | Regression_energy: 0.0783 | Regression_eta: 0.0687 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0713 | LR: 7.86e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2110s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank5 [2026-10-29 01:19:03,902] INFO : Step 16300/50000 rank5 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0772 | Regression_eta: 0.0534 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0732 | LR: 7.84e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.2093s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank5 [2026-10-29 01:23:56,182] INFO : Step 16400/50000 rank5 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2118 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0701 | Regression_eta: 0.0448 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0718 | LR: 7.81e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6829s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank5 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank5 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0669 | Regression_eta: 0.0449 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0875 | LR: 7.79e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3066s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank5 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank5 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1621 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0675 | Regression_eta: 0.0441 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0514 | LR: 7.77e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1708s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank5 [2026-10-29 01:38:38,619] INFO : Step 16700/50000 rank5 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2457 | Regression_cos_phi: 0.1188 | Regression_energy: 0.0823 | Regression_eta: 0.0709 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0677 | LR: 7.74e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9120s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank5 [2026-10-29 01:43:30,909] INFO : Step 16800/50000 rank5 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0909 | Regression_eta: 0.0682 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0944 | LR: 7.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.5998s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank5 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank5 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0995 | Regression_eta: 0.0349 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0879 | LR: 7.69e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3817s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank5 [2026-10-29 01:53:19,525] INFO : Step 17000/50000 rank5 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2322 | Regression_cos_phi: 0.1188 | Regression_energy: 0.0935 | Regression_eta: 0.0674 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0964 | LR: 7.67e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.6722s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank5 [2026-10-29 01:53:20,625] INFO : Running validation on rank5 +rank5 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 01:54:26,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:54:27,391] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 01:55:04,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 01:55:06,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 01:55:06,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 01:55:07,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 01:55:17,966] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:55:18,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 01:55:34,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 01:55:35,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 01:55:36,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 01:55:37,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 01:55:47,750] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:55:50,414] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 01:56:05,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 01:56:06,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 01:56:07,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 01:56:08,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 01:56:18,442] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 02:01:32,244] INFO : Step 17100/50000 rank5 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0677 | Regression_eta: 0.0481 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0663 | LR: 7.64e-04 | DataLoad Time: 0.0153s | Model Forward Time: 6.1173s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank5 [2026-10-29 02:06:24,500] INFO : Step 17200/50000 rank5 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0758 | Regression_eta: 0.0397 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0484 | LR: 7.62e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6222s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank5 [2026-10-29 02:11:06,516] INFO : Step 17300/50000 rank5 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0729 | Regression_eta: 0.0471 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0601 | LR: 7.59e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7523s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank5 [2026-10-29 02:16:07,513] INFO : Step 17400/50000 rank5 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0724 | Regression_eta: 0.0502 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0690 | LR: 7.57e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5463s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank5 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank5 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0856 | Regression_eta: 0.0370 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0653 | LR: 7.54e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.2409s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank5 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank5 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1478 | Regression_cos_phi: 0.1577 | Regression_energy: 0.0695 | Regression_eta: 0.0770 | Regression_pt: 0.0735 | Regression_sin_phi: 0.1021 | LR: 7.52e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4704s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank5 [2026-10-29 02:30:47,052] INFO : Step 17700/50000 rank5 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0832 | Regression_eta: 0.0717 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0764 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7704s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank5 [2026-10-29 02:35:37,696] INFO : Step 17800/50000 rank5 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.2401 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0721 | Regression_eta: 0.0571 | Regression_pt: 0.0736 | Regression_sin_phi: 0.1186 | LR: 7.47e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0209s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank5 [2026-10-29 02:40:30,747] INFO : Step 17900/50000 rank5 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0801 | Regression_eta: 0.0421 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0703 | LR: 7.44e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank5 [2026-10-29 02:45:16,094] INFO : Step 18000/50000 rank5 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2064 | Regression_cos_phi: 0.1136 | Regression_energy: 0.0732 | Regression_eta: 0.0712 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0924 | LR: 7.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6572s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank5 [2026-10-29 02:45:17,190] INFO : Running validation on rank5 +rank5 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:46:12,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:46:13,323] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 02:46:39,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:46:40,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:46:41,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:46:42,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:46:52,477] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:46:53,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 02:47:09,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:47:10,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:47:11,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:47:12,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:47:22,513] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:47:25,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:47:25,821] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 02:47:41,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:47:42,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:47:43,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:47:44,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:47:54,245] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank5 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0634 | Regression_eta: 0.0519 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0659 | LR: 7.39e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7377s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank5 [2026-10-29 02:58:08,644] INFO : Step 18200/50000 rank5 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0859 | Regression_eta: 0.0516 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0897 | LR: 7.36e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.9643s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank5 [2026-10-29 03:02:55,066] INFO : Step 18300/50000 rank5 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0780 | Regression_eta: 0.0605 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0962 | LR: 7.34e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.5930s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank5 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank5 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0645 | Regression_eta: 0.0505 | Regression_pt: 0.0680 | Regression_sin_phi: 0.1165 | LR: 7.31e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7124s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank5 [2026-10-29 03:12:43,564] INFO : Step 18500/50000 rank5 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0675 | Regression_eta: 0.0426 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0734 | LR: 7.29e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.3590s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank5 [2026-10-29 03:17:39,474] INFO : Step 18600/50000 rank5 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0715 | Regression_eta: 0.0482 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0685 | LR: 7.26e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4743s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank5 [2026-10-29 03:22:31,158] INFO : Step 18700/50000 rank5 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2085 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0856 | Regression_eta: 0.0495 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0758 | LR: 7.24e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4794s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank5 [2026-10-29 03:27:37,244] INFO : Step 18800/50000 rank5 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0608 | Regression_eta: 0.0373 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0753 | LR: 7.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.9534s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank5 [2026-10-29 03:32:26,943] INFO : Step 18900/50000 rank5 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0881 | Regression_eta: 0.0381 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0528 | LR: 7.18e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5369s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank5 [2026-10-29 03:37:06,568] INFO : Step 19000/50000 rank5 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0672 | Regression_eta: 0.0616 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0849 | LR: 7.16e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5309s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank5 [2026-10-29 03:37:07,713] INFO : Running validation on rank5 +rank5 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 03:38:00,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:38:00,820] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 03:38:27,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 03:38:28,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 03:38:29,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 03:38:30,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 03:38:40,200] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 03:38:40,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:38:41,429] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 03:38:57,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 03:38:58,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 03:38:59,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 03:39:00,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 03:39:10,409] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:39:11,231] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 03:39:27,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 03:39:27,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 03:39:28,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 03:39:29,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 03:39:39,723] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank5 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2551 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0673 | Regression_eta: 0.0578 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0678 | LR: 7.13e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9867s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank5 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank5 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0708 | Regression_eta: 0.0318 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0504 | LR: 7.10e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5598s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank5 [2026-10-29 03:54:09,158] INFO : Step 19300/50000 rank5 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1253 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0750 | Regression_eta: 0.0332 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0888 | LR: 7.08e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3957s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank5 [2026-10-29 03:58:51,216] INFO : Step 19400/50000 rank5 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0937 | Regression_eta: 0.0384 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0618 | LR: 7.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6150s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank5 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank5 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2333 | Regression_cos_phi: 0.1133 | Regression_energy: 0.0725 | Regression_eta: 0.0653 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0806 | LR: 7.02e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7166s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank5 [2026-10-29 04:08:12,465] INFO : Step 19600/50000 rank5 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0713 | Regression_eta: 0.0410 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0489 | LR: 7.00e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.4177s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank5 [2026-10-29 04:12:52,347] INFO : Step 19700/50000 rank5 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.2147 | Regression_cos_phi: 0.1007 | Regression_energy: 0.0764 | Regression_eta: 0.0528 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0783 | LR: 6.97e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3576s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank5 [2026-10-29 04:17:28,315] INFO : Step 19800/50000 rank5 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2382 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0608 | Regression_eta: 0.0550 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0949 | LR: 6.94e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0470s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank5 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank5 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2107 | Regression_cos_phi: 0.0769 | Regression_energy: 0.1020 | Regression_eta: 0.0713 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0905 | LR: 6.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3176s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank5 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank5 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0968 | Regression_eta: 0.0434 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0783 | LR: 6.89e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6547s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank5 [2026-10-29 04:26:52,970] INFO : Running validation on rank5 +rank5 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:27:46,036] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:27:46,652] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 04:28:14,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:28:15,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:28:16,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:28:17,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:28:27,460] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:28:28,501] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 04:28:44,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:28:45,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:28:46,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:28:47,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:28:57,964] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:28:59,249] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 04:29:14,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:29:15,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:29:16,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:29:17,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:29:27,330] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 04:34:27,055] INFO : Step 20100/50000 rank5 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2088 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0748 | Regression_eta: 0.0373 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0681 | LR: 6.86e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.3254s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank5 [2026-10-29 04:39:08,147] INFO : Step 20200/50000 rank5 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0364 | Regression_energy: 0.0751 | Regression_eta: 0.0378 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0672 | LR: 6.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6163s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank5 [2026-10-29 04:43:47,103] INFO : Step 20300/50000 rank5 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0552 | Regression_eta: 0.0477 | Regression_pt: 0.0557 | Regression_sin_phi: 0.1014 | LR: 6.81e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4961s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank5 [2026-10-29 04:48:26,732] INFO : Step 20400/50000 rank5 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2028 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0655 | Regression_eta: 0.0366 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0626 | LR: 6.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4428s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank5 [2026-10-29 04:53:08,468] INFO : Step 20500/50000 rank5 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0620 | Regression_eta: 0.0541 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0988 | LR: 6.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0641s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank5 [2026-10-29 04:57:53,295] INFO : Step 20600/50000 rank5 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0881 | Regression_eta: 0.0633 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0780 | LR: 6.73e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7514s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank5 [2026-10-29 05:02:47,890] INFO : Step 20700/50000 rank5 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0654 | Regression_eta: 0.0506 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0685 | LR: 6.70e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6385s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank5 [2026-10-29 05:07:34,620] INFO : Step 20800/50000 rank5 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0699 | Regression_eta: 0.0393 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0758 | LR: 6.67e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.8150s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank5 [2026-10-29 05:12:18,487] INFO : Step 20900/50000 rank5 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0790 | Regression_eta: 0.0469 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0768 | LR: 6.65e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6292s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank5 [2026-10-29 05:17:03,999] INFO : Step 21000/50000 rank5 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1320 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0647 | Regression_eta: 0.0469 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0776 | LR: 6.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5949s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank5 [2026-10-29 05:17:04,884] INFO : Running validation on rank5 +rank5 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 05:17:58,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:17:58,802] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 05:18:22,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 05:18:23,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 05:18:24,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 05:18:25,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 05:18:35,878] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:18:37,119] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 05:18:53,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 05:18:54,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 05:18:55,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 05:18:56,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 05:19:06,075] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 05:19:06,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:19:07,296] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 05:19:22,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 05:19:23,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 05:19:24,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 05:19:25,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 05:19:35,321] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank5 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0958 | Regression_energy: 0.0777 | Regression_eta: 0.0567 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0836 | LR: 6.59e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3803s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank5 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank5 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 05:29:24,522] INFO : Step 21200/50000 rank5 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1136 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0692 | Regression_eta: 0.0304 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0495 | LR: 6.56e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6431s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank5 [2026-10-29 05:34:39,115] INFO : Step 21300/50000 rank5 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0808 | Regression_eta: 0.0389 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0632 | LR: 6.54e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6727s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank5 [2026-10-29 05:39:29,690] INFO : Step 21400/50000 rank5 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0699 | Regression_eta: 0.0645 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0836 | LR: 6.51e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3965s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank5 [2026-10-29 05:44:17,736] INFO : Step 21500/50000 rank5 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0698 | Regression_eta: 0.0527 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0408 | LR: 6.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3104s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank5 [2026-10-29 05:49:11,033] INFO : Step 21600/50000 rank5 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0652 | Regression_eta: 0.0403 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0722 | LR: 6.45e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5085s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank5 [2026-10-29 05:53:59,566] INFO : Step 21700/50000 rank5 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0653 | Regression_eta: 0.0377 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0596 | LR: 6.43e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.5572s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank5 [2026-10-29 05:58:52,460] INFO : Step 21800/50000 rank5 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0695 | Regression_eta: 0.0295 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0639 | LR: 6.40e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.1501s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank5 [2026-10-29 06:03:49,408] INFO : Step 21900/50000 rank5 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0682 | Regression_eta: 0.0564 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0805 | LR: 6.37e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.8542s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank5 [2026-10-29 06:08:50,709] INFO : Step 22000/50000 rank5 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2441 | Regression_cos_phi: 0.1007 | Regression_energy: 0.0794 | Regression_eta: 0.0655 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0931 | LR: 6.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4729s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank5 [2026-10-29 06:08:51,802] INFO : Running validation on rank5 +rank5 [2026-10-29 06:08:51,803] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 06:08:51,803] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 06:09:45,573] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:09:46,124] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 06:10:12,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 06:10:13,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 06:10:14,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 06:10:15,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 06:10:25,507] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 06:10:26,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:10:27,416] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 06:10:43,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 06:10:44,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 06:10:45,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 06:10:46,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 06:10:56,338] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:10:56,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:10:57,445] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 06:11:13,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 06:11:13,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 06:11:14,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 06:11:15,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 06:11:25,687] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank5 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0659 | Regression_eta: 0.0578 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0991 | LR: 6.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1896s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank5 [2026-10-29 06:21:31,730] INFO : Step 22200/50000 rank5 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0675 | Regression_eta: 0.0316 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0491 | LR: 6.29e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.2922s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank5 [2026-10-29 06:26:33,413] INFO : Step 22300/50000 rank5 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0613 | Regression_eta: 0.0524 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0790 | LR: 6.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7528s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank5 [2026-10-29 06:31:29,808] INFO : Step 22400/50000 rank5 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0734 | Regression_eta: 0.0441 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0678 | LR: 6.23e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.4003s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank5 [2026-10-29 06:36:31,515] INFO : Step 22500/50000 rank5 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0663 | Regression_eta: 0.0596 | Regression_pt: 0.0706 | Regression_sin_phi: 0.1106 | LR: 6.20e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.2546s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank5 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank5 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0714 | Regression_eta: 0.0398 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0955 | LR: 6.18e-04 | DataLoad Time: 0.0261s | Model Forward Time: 4.3112s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank5 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank5 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0611 | Regression_eta: 0.0437 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0788 | LR: 6.15e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.9423s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank5 [2026-10-29 06:51:26,820] INFO : Step 22800/50000 rank5 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0601 | Regression_eta: 0.0353 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0574 | LR: 6.12e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.4067s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank5 [2026-10-29 06:56:23,353] INFO : Step 22900/50000 rank5 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0560 | Regression_eta: 0.0484 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0689 | LR: 6.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8107s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank5 [2026-10-29 07:01:33,057] INFO : Step 23000/50000 rank5 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0772 | Regression_eta: 0.0466 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0647 | LR: 6.06e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4760s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank5 [2026-10-29 07:01:34,091] INFO : Running validation on rank5 +rank5 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:01:34,091] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:02:28,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:02:28,853] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 07:02:49,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:02:50,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:02:51,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:02:52,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:03:02,702] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:03:03,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:03:03,699] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 07:03:19,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:03:20,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:03:21,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:03:23,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:03:33,038] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:03:33,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:03:34,173] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 07:03:50,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:03:50,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:03:51,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:03:52,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:04:02,605] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 07:09:13,971] INFO : Step 23100/50000 rank5 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.2050 | Regression_cos_phi: 0.1070 | Regression_energy: 0.0606 | Regression_eta: 0.0643 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0849 | LR: 6.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1800s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank5 [2026-10-29 07:14:12,719] INFO : Step 23200/50000 rank5 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0566 | Regression_eta: 0.0464 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0605 | LR: 6.01e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.6118s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank5 [2026-10-29 07:19:05,797] INFO : Step 23300/50000 rank5 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0715 | Regression_eta: 0.0508 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0534 | LR: 5.98e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8581s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank5 [2026-10-29 07:24:01,132] INFO : Step 23400/50000 rank5 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0604 | Regression_eta: 0.0492 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0581 | LR: 5.95e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0100s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank5 [2026-10-29 07:29:14,711] INFO : Step 23500/50000 rank5 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0592 | Regression_eta: 0.0330 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0713 | LR: 5.92e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.8336s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank5 [2026-10-29 07:34:22,319] INFO : Step 23600/50000 rank5 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0701 | Regression_eta: 0.0549 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0679 | LR: 5.90e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.3908s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank5 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank5 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0664 | Regression_eta: 0.0702 | Regression_pt: 0.0692 | Regression_sin_phi: 0.1017 | LR: 5.87e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.8145s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank5 [2026-10-29 07:44:46,684] INFO : Step 23800/50000 rank5 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0577 | Regression_eta: 0.0655 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0690 | LR: 5.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8787s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank5 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank5 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0643 | Regression_eta: 0.0394 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0717 | LR: 5.81e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8171s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank5 [2026-10-29 07:54:58,899] INFO : Step 24000/50000 rank5 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0624 | Regression_eta: 0.0494 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0746 | LR: 5.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8551s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank5 [2026-10-29 07:54:59,832] INFO : Running validation on rank5 +rank5 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:55:54,112] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 07:56:20,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:56:21,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:56:22,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:56:23,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:56:33,947] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:56:35,051] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 07:56:50,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:56:51,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:56:52,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:56:53,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:57:03,740] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:57:05,693] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 07:57:21,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:57:22,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:57:23,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:57:24,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:57:34,198] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 08:03:04,653] INFO : Step 24100/50000 rank5 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0637 | Regression_eta: 0.0403 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0569 | LR: 5.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5915s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank5 [2026-10-29 08:07:59,793] INFO : Step 24200/50000 rank5 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0610 | Regression_eta: 0.0448 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0554 | LR: 5.73e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0936s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank5 [2026-10-29 08:13:18,506] INFO : Step 24300/50000 rank5 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0672 | Regression_eta: 0.0576 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0832 | LR: 5.70e-04 | DataLoad Time: 0.0211s | Model Forward Time: 7.4895s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank5 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank5 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2257 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0764 | Regression_eta: 0.0589 | Regression_pt: 0.0781 | Regression_sin_phi: 0.1028 | LR: 5.67e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1582s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank5 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank5 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0617 | Regression_eta: 0.0589 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0642 | LR: 5.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.5206s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank5 [2026-10-29 08:28:18,514] INFO : Step 24600/50000 rank5 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0245 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0815 | Regression_eta: 0.0362 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0529 | LR: 5.61e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7047s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank5 [2026-10-29 08:33:21,874] INFO : Step 24700/50000 rank5 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0639 | Regression_eta: 0.0693 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0643 | LR: 5.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9274s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank5 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank5 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0708 | Regression_eta: 0.0460 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0720 | LR: 5.56e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2087s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank5 [2026-10-29 08:43:21,524] INFO : Step 24900/50000 rank5 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0647 | Regression_eta: 0.0410 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0595 | LR: 5.53e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.4723s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank5 [2026-10-29 08:48:25,836] INFO : Step 25000/50000 rank5 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0537 | Regression_eta: 0.0385 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0521 | LR: 5.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9536s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank5 [2026-10-29 08:48:26,771] INFO : Running validation on rank5 +rank5 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 08:49:20,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:49:21,193] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 08:49:51,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 08:49:53,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 08:49:54,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 08:49:55,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 08:50:05,506] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:50:06,729] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 08:50:23,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 08:50:24,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 08:50:24,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 08:50:25,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 08:50:35,966] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:50:36,987] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 08:50:52,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 08:50:53,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 08:50:54,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 08:50:55,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 08:51:05,770] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 08:56:18,538] INFO : Step 25100/50000 rank5 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0845 | Regression_eta: 0.0540 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0617 | LR: 5.47e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.3803s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank5 [2026-10-29 09:01:01,591] INFO : Step 25200/50000 rank5 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0483 | Regression_eta: 0.0456 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0468 | LR: 5.44e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.8769s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank5 [2026-10-29 09:05:50,665] INFO : Step 25300/50000 rank5 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0674 | Regression_eta: 0.0368 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0475 | LR: 5.42e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9793s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank5 [2026-10-29 09:10:30,565] INFO : Step 25400/50000 rank5 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0545 | Regression_eta: 0.0471 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0655 | LR: 5.39e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1706s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank5 [2026-10-29 09:15:11,238] INFO : Step 25500/50000 rank5 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0596 | Regression_eta: 0.0453 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0455 | LR: 5.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4702s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank5 [2026-10-29 09:19:56,597] INFO : Step 25600/50000 rank5 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0679 | Regression_eta: 0.0436 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0424 | LR: 5.33e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.9354s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank5 [2026-10-29 09:24:33,865] INFO : Step 25700/50000 rank5 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0587 | Regression_eta: 0.0471 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0729 | LR: 5.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2134s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank5 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank5 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0662 | Regression_eta: 0.0419 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0720 | LR: 5.27e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.1678s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank5 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank5 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1204 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0585 | Regression_eta: 0.0387 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0690 | LR: 5.25e-04 | DataLoad Time: 0.0243s | Model Forward Time: 4.7712s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank5 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank5 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0635 | Regression_eta: 0.0403 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0625 | LR: 5.22e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7492s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank5 [2026-10-29 09:39:04,390] INFO : Running validation on rank5 +rank5 [2026-10-29 09:39:04,390] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:39:04,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:39:55,909] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 09:40:23,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:40:24,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:40:25,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:40:26,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:40:36,307] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:40:37,505] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 09:40:54,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:40:56,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:40:57,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:40:58,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:41:08,849] INFO : Time taken to make predictions on device 5 is: 0.29 min +rank5 [2026-10-29 09:41:08,937] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:44:09,566] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 09:44:25,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:44:26,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:44:27,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:44:28,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:44:38,145] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 09:50:29,555] INFO : Step 26100/50000 rank5 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0683 | Regression_eta: 0.0441 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0727 | LR: 5.19e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1784s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank5 [2026-10-29 09:55:15,274] INFO : Step 26200/50000 rank5 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0656 | Regression_eta: 0.0409 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0588 | LR: 5.16e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8145s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank5 [2026-10-29 09:59:57,745] INFO : Step 26300/50000 rank5 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0588 | Regression_eta: 0.0302 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0599 | LR: 5.13e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.9318s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank5 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank5 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0735 | Regression_eta: 0.0595 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0646 | LR: 5.10e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.1491s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank5 [2026-10-29 10:09:23,740] INFO : Step 26500/50000 rank5 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0596 | Regression_eta: 0.0353 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0462 | LR: 5.08e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.0940s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank5 [2026-10-29 10:14:05,781] INFO : Step 26600/50000 rank5 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0512 | Regression_eta: 0.0399 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0554 | LR: 5.05e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6627s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank5 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank5 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0584 | Regression_eta: 0.0421 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0627 | LR: 5.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2142s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank5 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank5 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0548 | Regression_eta: 0.0409 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0622 | LR: 4.99e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5308s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank5 [2026-10-29 10:28:51,387] INFO : Step 26900/50000 rank5 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0522 | Regression_eta: 0.0391 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0491 | LR: 4.96e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.5625s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank5 [2026-10-29 10:33:37,219] INFO : Step 27000/50000 rank5 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0565 | Regression_eta: 0.0449 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0965 | LR: 4.94e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.6923s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank5 [2026-10-29 10:33:38,215] INFO : Running validation on rank5 +rank5 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 10:34:33,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:34:34,220] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 10:35:00,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 10:35:01,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 10:35:02,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 10:35:03,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 10:35:13,126] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 10:35:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:35:14,237] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 10:35:30,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 10:35:31,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 10:35:32,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 10:35:33,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 10:35:43,673] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 10:35:44,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:35:45,037] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 10:36:00,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 10:36:01,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 10:36:02,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 10:36:03,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 10:36:13,121] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 10:41:28,284] INFO : Step 27100/50000 rank5 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0552 | Regression_eta: 0.0366 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0581 | LR: 4.91e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6756s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank5 [2026-10-29 10:46:22,293] INFO : Step 27200/50000 rank5 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0724 | Regression_eta: 0.0383 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0465 | LR: 4.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8260s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank5 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank5 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1079 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0570 | Regression_eta: 0.0257 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0657 | LR: 4.85e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6700s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank5 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank5 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0565 | Regression_eta: 0.0414 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0671 | LR: 4.82e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9759s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank5 [2026-10-29 11:01:35,254] INFO : Step 27500/50000 rank5 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0647 | Regression_eta: 0.0408 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0613 | LR: 4.80e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.3150s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank5 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank5 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0525 | Regression_eta: 0.0328 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0534 | LR: 4.77e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.3324s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank5 [2026-10-29 11:13:33,057] INFO : Step 27700/50000 rank5 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0546 | Regression_eta: 0.0458 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0710 | LR: 4.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7946s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank5 [2026-10-29 11:18:30,826] INFO : Step 27800/50000 rank5 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0551 | Regression_eta: 0.0369 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0500 | LR: 4.71e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0104s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank5 [2026-10-29 11:23:38,363] INFO : Step 27900/50000 rank5 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0647 | Regression_eta: 0.0472 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0407 | LR: 4.68e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8864s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank5 [2026-10-29 11:28:54,872] INFO : Step 28000/50000 rank5 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0622 | Regression_eta: 0.0475 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0569 | LR: 4.66e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.9103s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank5 [2026-10-29 11:28:55,940] INFO : Running validation on rank5 +rank5 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 11:29:47,987] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:29:48,521] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 11:30:14,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 11:30:15,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 11:30:16,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 11:30:17,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 11:30:27,452] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:30:28,727] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 11:30:44,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 11:30:45,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 11:30:46,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 11:30:47,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 11:30:57,949] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 11:30:57,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 11:31:13,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 11:31:14,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 11:31:15,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 11:31:16,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 11:31:26,542] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 11:37:01,726] INFO : Step 28100/50000 rank5 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0562 | Regression_eta: 0.0305 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0461 | LR: 4.63e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.5009s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank5 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank5 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0586 | Regression_eta: 0.0366 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0562 | LR: 4.60e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.0738s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank5 [2026-10-29 11:47:09,276] INFO : Step 28300/50000 rank5 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0547 | Regression_eta: 0.0392 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0410 | LR: 4.57e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.0926s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank5 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank5 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1644 | Classification_binary: 0.2896 | Regression_cos_phi: 0.1136 | Regression_energy: 0.1331 | Regression_eta: 0.0793 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0714 | LR: 4.55e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5922s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank5 [2026-10-29 11:56:56,841] INFO : Step 28500/50000 rank5 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0563 | Regression_eta: 0.0501 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0605 | LR: 4.52e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.9882s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank5 [2026-10-29 12:01:46,935] INFO : Step 28600/50000 rank5 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0586 | Regression_eta: 0.0335 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0540 | LR: 4.49e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6721s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank5 [2026-10-29 12:06:39,994] INFO : Step 28700/50000 rank5 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0517 | Regression_eta: 0.0372 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0429 | LR: 4.46e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.5045s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank5 [2026-10-29 12:11:31,024] INFO : Step 28800/50000 rank5 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0640 | Regression_eta: 0.0354 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0721 | LR: 4.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.9314s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank5 [2026-10-29 12:16:17,098] INFO : Step 28900/50000 rank5 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0643 | Regression_eta: 0.0356 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0458 | LR: 4.41e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7021s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank5 [2026-10-29 12:21:10,134] INFO : Step 29000/50000 rank5 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1200 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0584 | Regression_eta: 0.0355 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0595 | LR: 4.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9937s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank5 [2026-10-29 12:21:11,231] INFO : Running validation on rank5 +rank5 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:22:04,987] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 12:22:31,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 12:22:32,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 12:22:33,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 12:22:35,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 12:22:45,064] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:22:45,974] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 12:23:01,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 12:23:02,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 12:23:03,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 12:23:04,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 12:23:14,729] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 12:23:15,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:23:16,447] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 12:23:32,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 12:23:32,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 12:23:33,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 12:23:34,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 12:23:44,564] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank5 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1918 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0679 | Regression_eta: 0.0492 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0654 | LR: 4.35e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1728s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank5 [2026-10-29 12:33:43,194] INFO : Step 29200/50000 rank5 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0629 | Regression_eta: 0.0385 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0787 | LR: 4.33e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5655s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank5 [2026-10-29 14:49:26,334] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-29 14:49:26,350] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank5 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-29 14:49:54,088] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank5 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank5 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank5 [2026-10-29 14:49:54,132] INFO : Initializing backbone layer 2 +rank5 [2026-10-29 14:49:54,140] INFO : Initializing backbone layer 3 +rank5 [2026-10-29 14:49:54,149] INFO : Initializing backbone layer 4 +rank5 [2026-10-29 14:49:54,157] INFO : Initializing backbone layer 5 +rank5 [2026-10-29 14:49:54,165] INFO : Backbone initialization took 0.08s +rank5 [2026-10-29 14:49:54,166] INFO : backbone parameters: 7901376 +rank5 [2026-10-29 14:49:54,166] INFO : Initializing output DNNs +rank5 [2026-10-29 14:49:54,202] INFO : Output DNNs initialization took 0.04s +rank5 [2026-10-29 14:49:54,202] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-29 14:49:54,202] INFO : jagged_attention enabled=False +rank5 [2026-10-29 14:49:54,202] INFO : flash_attn_varlen enabled=False +rank5 [2026-10-29 14:49:54,202] INFO : task_queries enabled=False +rank5 [2026-10-29 14:49:54,202] INFO : classification_norm parameters: 0 +rank5 [2026-10-29 14:49:54,202] INFO : regression_norm parameters: 0 +rank5 [2026-10-29 14:49:54,202] INFO : classification_query parameters: 0 +rank5 [2026-10-29 14:49:54,202] INFO : regression_query parameters: 0 +rank5 [2026-10-29 14:49:54,202] INFO : classification_readout parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : regression_readout parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : output_mode=set +rank5 [2026-10-29 14:49:54,203] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-29 14:49:54,203] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_pid parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_pt parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_eta parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-29 14:49:54,203] INFO : nn_energy parameters: 0 +rank5 [2026-10-29 14:49:54,204] INFO : Total MLPF parameters: 12786964 +rank5 [2026-10-29 14:49:54,204] INFO : MLPF __init__ done +rank5 [2026-10-29 14:49:54,204] INFO : Instantiated model +rank5 [2026-10-29 14:49:54,204] INFO : Moving model to device rank=5 +rank5 [2026-10-29 14:49:54,225] INFO : Moved model to device rank=5 +rank5 [2026-10-29 14:49:54,227] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-29 14:49:54,793] INFO : Restoring training from step 29001 +rank5 [2026-10-29 14:49:54,801] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank5 [2026-10-29 14:49:54,801] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-29 14:49:55,511] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank5 [2026-10-29 14:49:55,511] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-29 14:49:55,513] INFO : Trainable parameters: 12786964 +rank5 [2026-10-29 14:49:55,513] INFO : Non-trainable parameters: 23268 +rank5 [2026-10-29 14:49:55,513] INFO : Total parameters: 12810232 +rank5 [2026-10-29 14:49:55,516] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-29 14:49:55,516] INFO : Getting dataloaders +rank5 [2026-10-29 14:49:55,517] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:01,368] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:02,347] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:03,540] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-29 14:50:03,541] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-29 14:50:03,541] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:03,807] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:04,069] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:50:04,319] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-29 14:50:04,319] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-29 14:50:18,100] INFO : Got dataloaders +rank5 [2026-10-29 14:50:18,100] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank5 [2026-10-29 14:50:18,100] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank5 [2026-10-29 14:50:18,100] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank5 [2026-10-29 14:50:18,100] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank5 [2026-10-29 14:50:19,733] INFO : loader split=train rank=5 len=21108 +rank5 [2026-10-29 14:50:19,733] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-29 14:50:19,733] INFO : Starting training from step 29001 to 50000 on rank 5 of 8 +rank5 [2026-10-29 14:50:19,733] INFO : Creating train_iterator +rank5 [2026-10-29 14:50:19,733] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank5 [2026-10-29 14:50:33,577] INFO : Created train_iterator +rank5 [2026-10-29 14:55:12,920] INFO : Step 29100/50000 rank5 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0680 | Regression_eta: 0.0391 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0537 | LR: 4.35e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7165s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank5 [2026-10-29 14:59:46,788] INFO : Step 29200/50000 rank5 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0601 | Regression_eta: 0.0377 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0554 | LR: 4.33e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3622s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank5 [2026-10-29 15:04:21,834] INFO : Step 29300/50000 rank5 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0488 | Regression_eta: 0.0261 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0820 | LR: 4.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0924s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank5 [2026-10-29 15:08:57,032] INFO : Step 29400/50000 rank5 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0734 | Regression_eta: 0.0494 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0637 | LR: 4.27e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2740s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank5 [2026-10-29 15:13:31,011] INFO : Step 29500/50000 rank5 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0589 | Regression_eta: 0.0435 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0792 | LR: 4.24e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4985s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank5 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank5 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0744 | Regression_eta: 0.0317 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0536 | LR: 4.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5922s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank5 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank5 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0529 | Regression_eta: 0.0413 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0655 | LR: 4.19e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6723s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank5 [2026-10-29 15:27:18,563] INFO : Step 29800/50000 rank5 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2291 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0572 | Regression_eta: 0.0757 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0809 | LR: 4.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6303s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank5 [2026-10-29 15:31:51,912] INFO : Step 29900/50000 rank5 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0628 | Regression_eta: 0.0562 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0754 | LR: 4.14e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5839s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank5 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank5 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0583 | Regression_eta: 0.0406 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0709 | LR: 4.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4513s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank5 [2026-10-29 15:36:27,739] INFO : Running validation on rank5 +rank5 [2026-10-29 15:36:27,740] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 15:36:27,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 15:37:19,265] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 15:37:50,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 15:37:51,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 15:37:52,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 15:37:53,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 15:38:03,828] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:38:04,946] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 15:38:20,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 15:38:21,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 15:38:22,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 15:38:23,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 15:38:33,615] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:38:35,047] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 15:38:50,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 15:38:51,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 15:38:52,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 15:38:53,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 15:39:03,072] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank5 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0603 | Regression_eta: 0.0336 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0713 | LR: 4.08e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7114s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank5 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank5 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0543 | Regression_eta: 0.0296 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0507 | LR: 4.06e-04 | DataLoad Time: 0.0272s | Model Forward Time: 4.7623s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank5 [2026-10-29 15:53:38,867] INFO : Step 30300/50000 rank5 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0530 | Regression_eta: 0.0375 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0644 | LR: 4.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3377s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank5 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank5 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0605 | Regression_eta: 0.0415 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0848 | LR: 4.00e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4509s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank5 [2026-10-29 16:02:51,487] INFO : Step 30500/50000 rank5 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2174 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0892 | Regression_eta: 0.0504 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0690 | LR: 3.98e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4573s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank5 [2026-10-29 16:07:32,049] INFO : Step 30600/50000 rank5 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0557 | Regression_eta: 0.0286 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0490 | LR: 3.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9371s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank5 [2026-10-29 16:12:18,779] INFO : Step 30700/50000 rank5 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0590 | Regression_eta: 0.0289 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0627 | LR: 3.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2827s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank5 [2026-10-29 16:17:03,636] INFO : Step 30800/50000 rank5 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0540 | Regression_eta: 0.0417 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0972 | LR: 3.90e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0644s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank5 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank5 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0579 | Regression_eta: 0.0403 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0811 | LR: 3.87e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6948s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank5 [2026-10-29 16:26:02,768] INFO : Step 31000/50000 rank5 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0613 | Regression_eta: 0.0327 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0658 | LR: 3.84e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0245s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank5 [2026-10-29 16:26:49,214] INFO : Running validation on rank5 +rank5 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:27:39,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:27:40,308] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 16:28:08,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:28:09,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:28:11,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:28:13,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:28:23,048] INFO : Time taken to make predictions on device 5 is: 0.29 min +rank5 [2026-10-29 16:28:24,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:28:24,725] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 16:28:40,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:28:41,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:28:42,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:28:43,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:28:53,888] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:28:54,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:28:54,677] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 16:29:10,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:29:11,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:29:12,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:29:13,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:29:23,072] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank5 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0474 | Regression_eta: 0.0476 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0535 | LR: 3.82e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.5522s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank5 [2026-10-29 16:39:13,625] INFO : Step 31200/50000 rank5 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0695 | Regression_eta: 0.0420 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0436 | LR: 3.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0723s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank5 [2026-10-29 16:43:57,174] INFO : Step 31300/50000 rank5 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1168 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0398 | Regression_eta: 0.0288 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0370 | LR: 3.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8834s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank5 [2026-10-29 16:48:30,667] INFO : Step 31400/50000 rank5 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0573 | Regression_eta: 0.0446 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0634 | LR: 3.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7850s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank5 [2026-10-29 16:53:07,752] INFO : Step 31500/50000 rank5 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0682 | Regression_eta: 0.0418 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0653 | LR: 3.71e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4921s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank5 [2026-10-29 16:57:38,608] INFO : Step 31600/50000 rank5 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1322 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0525 | Regression_eta: 0.0469 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0694 | LR: 3.69e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1671s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank5 [2026-10-29 17:02:11,476] INFO : Step 31700/50000 rank5 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1258 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0462 | Regression_eta: 0.0316 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0754 | LR: 3.66e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1025s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank5 [2026-10-29 17:06:47,526] INFO : Step 31800/50000 rank5 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0662 | Regression_eta: 0.0587 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0955 | LR: 3.64e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2362s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank5 [2026-10-29 17:11:27,373] INFO : Step 31900/50000 rank5 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0490 | Regression_eta: 0.0304 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0505 | LR: 3.61e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5190s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank5 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank5 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0584 | Regression_eta: 0.0535 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0532 | LR: 3.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4955s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank5 [2026-10-29 17:16:12,936] INFO : Running validation on rank5 +rank5 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 17:17:03,855] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:17:04,362] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 17:17:32,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 17:17:33,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 17:17:33,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 17:17:34,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 17:17:44,996] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:17:46,318] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 17:18:02,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 17:18:03,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 17:18:04,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 17:18:05,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 17:18:15,286] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:18:15,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:18:16,465] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 17:18:32,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 17:18:33,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 17:18:33,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 17:18:34,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 17:18:44,813] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank5 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0506 | Regression_eta: 0.0329 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0380 | LR: 3.56e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9464s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank5 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank5 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0565 | Regression_eta: 0.0398 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0585 | LR: 3.53e-04 | DataLoad Time: 0.0137s | Model Forward Time: 5.4481s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank5 [2026-10-29 17:33:21,418] INFO : Step 32300/50000 rank5 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0542 | Regression_eta: 0.0497 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0472 | LR: 3.51e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.9290s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank5 [2026-10-29 17:38:08,351] INFO : Step 32400/50000 rank5 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0502 | Regression_eta: 0.0346 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0614 | LR: 3.48e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8166s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank5 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank5 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0606 | Regression_eta: 0.0238 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0354 | LR: 3.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8580s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank5 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank5 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0546 | Regression_eta: 0.0383 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0685 | LR: 3.43e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.6196s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank5 [2026-10-29 17:52:17,189] INFO : Step 32700/50000 rank5 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1188 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0483 | Regression_eta: 0.0366 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0578 | LR: 3.41e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5871s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank5 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank5 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0508 | Regression_eta: 0.0303 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0375 | LR: 3.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5892s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank5 [2026-10-29 18:01:38,149] INFO : Step 32900/50000 rank5 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0500 | Regression_eta: 0.0499 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0698 | LR: 3.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4257s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank5 [2026-10-29 18:06:22,611] INFO : Step 33000/50000 rank5 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0536 | Regression_eta: 0.0397 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0583 | LR: 3.33e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2122s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank5 [2026-10-29 18:06:23,517] INFO : Running validation on rank5 +rank5 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:07:15,766] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 18:07:43,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:07:44,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:07:45,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:07:46,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:07:56,948] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:07:58,401] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 18:08:14,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:08:15,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:08:16,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:08:17,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:08:27,564] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:08:28,344] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 18:08:43,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:08:44,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:08:45,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:08:46,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:08:56,472] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank5 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1291 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0456 | Regression_eta: 0.0365 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0624 | LR: 3.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2605s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank5 [2026-10-29 18:18:33,982] INFO : Step 33200/50000 rank5 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0456 | Regression_eta: 0.0338 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0747 | LR: 3.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4344s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank5 [2026-10-29 18:23:15,421] INFO : Step 33300/50000 rank5 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0245 | Classification_binary: 0.1056 | Regression_cos_phi: 0.0280 | Regression_energy: 0.0542 | Regression_eta: 0.0221 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0280 | LR: 3.26e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6024s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank5 [2026-10-29 18:27:59,291] INFO : Step 33400/50000 rank5 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0639 | Regression_eta: 0.0473 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0618 | LR: 3.23e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.2645s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank5 [2026-10-29 18:32:38,324] INFO : Step 33500/50000 rank5 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0565 | Regression_eta: 0.0371 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0527 | LR: 3.21e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3732s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank5 [2026-10-29 18:37:27,753] INFO : Step 33600/50000 rank5 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0538 | Regression_eta: 0.0480 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0516 | LR: 3.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3472s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank5 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank5 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0586 | Regression_eta: 0.0461 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0466 | LR: 3.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3195s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank5 [2026-10-29 18:47:04,986] INFO : Step 33800/50000 rank5 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0568 | Regression_eta: 0.0360 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0534 | LR: 3.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3945s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank5 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank5 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0493 | Regression_eta: 0.0542 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0535 | LR: 3.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1937s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank5 [2026-10-29 18:56:35,808] INFO : Step 34000/50000 rank5 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.0997 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0564 | Regression_eta: 0.0239 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0521 | LR: 3.09e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1635s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank5 [2026-10-29 18:56:36,807] INFO : Running validation on rank5 +rank5 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:57:29,827] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:57:30,355] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 18:57:54,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:57:55,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:57:56,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:57:57,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:58:07,953] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:58:09,211] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 18:58:25,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:58:26,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:58:27,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:58:28,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:58:38,514] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:58:39,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 18:58:55,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:58:56,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:58:57,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:58:57,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:59:07,996] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 19:04:16,498] INFO : Step 34100/50000 rank5 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0503 | Regression_eta: 0.0477 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0502 | LR: 3.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8505s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank5 [2026-10-29 19:09:14,637] INFO : Step 34200/50000 rank5 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0548 | Regression_eta: 0.0422 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0424 | LR: 3.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6994s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank5 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank5 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0212 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0474 | Regression_eta: 0.0312 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0424 | LR: 3.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3183s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank5 [2026-10-29 19:18:48,203] INFO : Step 34400/50000 rank5 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0463 | Regression_eta: 0.0257 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0591 | LR: 2.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9243s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank5 [2026-10-29 19:23:49,388] INFO : Step 34500/50000 rank5 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0478 | Regression_eta: 0.0314 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0460 | LR: 2.97e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.2317s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank5 [2026-10-29 19:28:34,774] INFO : Step 34600/50000 rank5 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0554 | Regression_eta: 0.0449 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0507 | LR: 2.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4731s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank5 [2026-10-29 19:33:35,743] INFO : Step 34700/50000 rank5 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0388 | Regression_eta: 0.0326 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0560 | LR: 2.92e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1496s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank5 [2026-10-29 19:38:26,589] INFO : Step 34800/50000 rank5 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1419 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0464 | Regression_eta: 0.0457 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0686 | LR: 2.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0549s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank5 [2026-10-29 19:43:15,573] INFO : Step 34900/50000 rank5 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0492 | Regression_eta: 0.0269 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0380 | LR: 2.88e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2756s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank5 [2026-10-29 19:48:20,348] INFO : Step 35000/50000 rank5 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0286 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0514 | Regression_eta: 0.0375 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0564 | LR: 2.85e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3852s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank5 [2026-10-29 19:48:21,335] INFO : Running validation on rank5 +rank5 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 19:49:15,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:49:15,988] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 19:49:42,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 19:49:43,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 19:49:44,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 19:49:45,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 19:49:55,517] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:49:56,736] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 19:50:13,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 19:50:14,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 19:50:14,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 19:50:15,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 19:50:25,972] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:50:26,883] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 19:50:42,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 19:50:43,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 19:50:43,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 19:50:44,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 19:50:54,868] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 19:56:07,675] INFO : Step 35100/50000 rank5 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0500 | Regression_eta: 0.0315 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0505 | LR: 2.83e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4696s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank5 [2026-10-29 20:01:01,619] INFO : Step 35200/50000 rank5 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0288 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0477 | Regression_eta: 0.0328 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0649 | LR: 2.81e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6721s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank5 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank5 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0588 | Regression_eta: 0.0351 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0514 | LR: 2.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1455s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank5 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank5 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0586 | Regression_eta: 0.0288 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0623 | LR: 2.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4645s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank5 [2026-10-29 20:15:42,290] INFO : Step 35500/50000 rank5 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0548 | Regression_eta: 0.0402 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0725 | LR: 2.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3583s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank5 [2026-10-29 20:20:38,046] INFO : Step 35600/50000 rank5 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0535 | Regression_eta: 0.0324 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0410 | LR: 2.72e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1894s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank5 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank5 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0473 | Regression_eta: 0.0274 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0457 | LR: 2.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1731s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank5 [2026-10-29 20:30:22,090] INFO : Step 35800/50000 rank5 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0436 | Regression_eta: 0.0317 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0438 | LR: 2.68e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2788s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank5 [2026-10-29 20:35:20,188] INFO : Step 35900/50000 rank5 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0617 | Regression_eta: 0.0401 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0592 | LR: 2.65e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6686s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank5 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank5 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0564 | Regression_eta: 0.0308 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0480 | LR: 2.63e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2579s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank5 [2026-10-29 20:40:17,059] INFO : Running validation on rank5 +rank5 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 20:40:17,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 20:41:08,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:41:08,960] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 20:41:35,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 20:41:36,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 20:41:37,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 20:41:38,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 20:41:48,875] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:41:50,340] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 20:42:05,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 20:42:06,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 20:42:07,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 20:42:08,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 20:42:18,848] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:42:20,694] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 20:42:36,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 20:42:36,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 20:42:37,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 20:42:38,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 20:42:48,589] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 20:47:46,989] INFO : Step 36100/50000 rank5 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.0962 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0601 | Regression_eta: 0.0349 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0384 | LR: 2.61e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4504s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank5 [2026-10-29 20:52:22,843] INFO : Step 36200/50000 rank5 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0444 | Regression_eta: 0.0446 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0581 | LR: 2.59e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3985s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank5 [2026-10-29 20:56:59,585] INFO : Step 36300/50000 rank5 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0611 | Regression_eta: 0.0322 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0502 | LR: 2.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2281s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank5 [2026-10-29 21:01:38,352] INFO : Step 36400/50000 rank5 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0485 | Regression_eta: 0.0285 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0507 | LR: 2.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8762s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank5 [2026-10-29 21:06:14,246] INFO : Step 36500/50000 rank5 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0457 | Regression_eta: 0.0392 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0383 | LR: 2.52e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4227s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank5 [2026-10-29 21:10:49,733] INFO : Step 36600/50000 rank5 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0597 | Regression_eta: 0.0321 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0523 | LR: 2.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2774s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank5 [2026-10-29 21:15:35,109] INFO : Step 36700/50000 rank5 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0387 | Regression_eta: 0.0343 | Regression_pt: 0.0404 | Regression_sin_phi: 0.0683 | LR: 2.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2705s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank5 [2026-10-29 21:20:28,004] INFO : Step 36800/50000 rank5 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1256 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0665 | Regression_eta: 0.0313 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0424 | LR: 2.46e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3997s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank5 [2026-10-29 21:25:23,683] INFO : Step 36900/50000 rank5 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0505 | Regression_eta: 0.0350 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0670 | LR: 2.44e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9373s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank5 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank5 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0622 | Regression_eta: 0.0399 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0487 | LR: 2.42e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9284s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank5 [2026-10-29 21:30:17,323] INFO : Running validation on rank5 +rank5 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:31:08,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:31:08,621] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 21:31:35,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:31:36,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:31:37,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:31:38,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:31:48,308] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:31:49,070] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:31:49,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 21:32:05,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:32:07,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:32:08,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:32:09,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:32:19,484] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:32:20,812] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 21:32:36,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:32:37,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:32:38,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:32:39,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:32:49,302] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 21:37:56,197] INFO : Step 37100/50000 rank5 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1252 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0438 | Regression_eta: 0.0330 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0585 | LR: 2.40e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6901s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank5 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank5 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0481 | Regression_eta: 0.0282 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0529 | LR: 2.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6962s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank5 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank5 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1159 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0517 | Regression_eta: 0.0367 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0627 | LR: 2.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5591s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank5 [2026-10-29 21:52:56,910] INFO : Step 37400/50000 rank5 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0284 | Classification_binary: 0.0958 | Regression_cos_phi: 0.0317 | Regression_energy: 0.0407 | Regression_eta: 0.0334 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0699 | LR: 2.34e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.6148s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank5 [2026-10-29 21:57:43,877] INFO : Step 37500/50000 rank5 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0514 | Regression_eta: 0.0256 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0352 | LR: 2.32e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7111s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank5 [2026-10-29 22:02:29,840] INFO : Step 37600/50000 rank5 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0439 | Regression_eta: 0.0249 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0509 | LR: 2.30e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2477s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank5 [2026-10-29 22:07:05,609] INFO : Step 37700/50000 rank5 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0431 | Regression_eta: 0.0387 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0715 | LR: 2.28e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1550s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank5 [2026-10-29 22:11:38,776] INFO : Step 37800/50000 rank5 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0546 | Regression_eta: 0.0434 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0796 | LR: 2.26e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank5 [2026-10-29 22:16:15,814] INFO : Step 37900/50000 rank5 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0544 | Regression_eta: 0.0335 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0609 | LR: 2.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1266s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank5 [2026-10-29 22:20:56,949] INFO : Step 38000/50000 rank5 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0575 | Regression_eta: 0.0502 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0569 | LR: 2.22e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2885s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank5 [2026-10-29 22:20:57,876] INFO : Running validation on rank5 +rank5 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 22:20:57,877] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 22:21:48,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:21:48,922] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 22:22:15,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 22:22:16,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 22:22:17,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 22:22:18,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 22:22:28,177] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 22:22:28,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:22:29,428] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 22:22:45,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 22:22:46,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 22:22:47,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 22:22:48,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 22:22:58,289] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:22:59,394] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 22:23:14,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 22:23:15,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 22:23:16,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 22:23:17,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 22:23:27,208] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank5 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.0961 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0515 | Regression_eta: 0.0291 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0536 | LR: 2.20e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2304s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank5 [2026-10-29 22:33:07,241] INFO : Step 38200/50000 rank5 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0390 | Regression_eta: 0.0369 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0477 | LR: 2.18e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.7620s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank5 [2026-10-29 22:37:42,243] INFO : Step 38300/50000 rank5 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0460 | Regression_eta: 0.0523 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0947 | LR: 2.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5934s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank5 [2026-10-29 22:42:17,609] INFO : Step 38400/50000 rank5 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0494 | Regression_eta: 0.0385 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0353 | LR: 2.14e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4885s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank5 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank5 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1083 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0529 | Regression_eta: 0.0349 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0374 | LR: 2.12e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5217s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank5 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank5 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1634 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0509 | Regression_eta: 0.0311 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0516 | LR: 2.11e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7263s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank5 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank5 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0556 | Regression_eta: 0.0308 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0562 | LR: 2.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1564s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank5 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank5 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1220 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0608 | Regression_eta: 0.0262 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0327 | LR: 2.07e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3976s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank5 [2026-10-29 23:07:29,433] INFO : Step 38900/50000 rank5 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0487 | Regression_eta: 0.0320 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0425 | LR: 2.05e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8883s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank5 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank5 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0386 | Regression_eta: 0.0432 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0591 | LR: 2.03e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4692s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank5 [2026-10-29 23:12:38,866] INFO : Running validation on rank5 +rank5 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 23:13:33,721] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:13:34,229] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-29 23:14:06,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 23:14:07,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 23:14:08,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 23:14:09,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 23:14:19,525] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 23:14:20,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:14:20,841] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-29 23:14:36,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 23:14:37,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 23:14:38,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 23:14:39,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 23:14:49,800] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:14:50,899] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-29 23:15:06,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 23:15:07,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 23:15:08,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 23:15:08,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 23:15:18,981] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank5 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0411 | Regression_eta: 0.0262 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0479 | LR: 2.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9857s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank5 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank5 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0417 | Regression_eta: 0.0283 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0304 | LR: 2.00e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5908s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank5 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank5 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0433 | Regression_eta: 0.0422 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0360 | LR: 1.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6306s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank5 [2026-10-29 23:34:45,056] INFO : Step 39400/50000 rank5 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1105 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0412 | Regression_eta: 0.0334 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0540 | LR: 1.96e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.6871s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank5 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank5 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1028 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0431 | Regression_eta: 0.0271 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0624 | LR: 1.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4400s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank5 [2026-10-29 23:43:57,879] INFO : Step 39600/50000 rank5 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0424 | Regression_eta: 0.0374 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0503 | LR: 1.93e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4676s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank5 [2026-10-29 23:48:42,957] INFO : Step 39700/50000 rank5 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0390 | Regression_eta: 0.0283 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0462 | LR: 1.91e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4038s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank5 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank5 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0289 | Classification_binary: 0.0982 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0406 | Regression_eta: 0.0151 | Regression_pt: 0.0408 | Regression_sin_phi: 0.0381 | LR: 1.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5381s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank5 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank5 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0489 | Regression_eta: 0.0365 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0483 | LR: 1.88e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7661s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank5 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank5 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0635 | Regression_eta: 0.0262 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0515 | LR: 1.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9278s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank5 [2026-10-30 00:05:56,736] INFO : Running validation on rank5 +rank5 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:06:50,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:06:51,354] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 00:07:17,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:07:18,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:07:19,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:07:20,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:07:30,716] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:07:31,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 00:07:47,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:07:48,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:07:49,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:07:50,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:08:00,600] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:08:01,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:08:02,100] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 00:08:17,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 00:08:18,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 00:08:19,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 00:08:20,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 00:08:30,284] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 00:13:50,202] INFO : Step 40100/50000 rank5 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0656 | Regression_eta: 0.0322 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0538 | LR: 1.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6423s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank5 [2026-10-30 00:18:43,889] INFO : Step 40200/50000 rank5 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0416 | Regression_eta: 0.0298 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0484 | LR: 1.83e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4469s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank5 [2026-10-30 00:23:35,704] INFO : Step 40300/50000 rank5 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0535 | Regression_eta: 0.0319 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0630 | LR: 1.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5345s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank5 [2026-10-30 00:28:38,368] INFO : Step 40400/50000 rank5 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0470 | Regression_eta: 0.0525 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0561 | LR: 1.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2212s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank5 [2026-10-30 00:33:25,153] INFO : Step 40500/50000 rank5 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1200 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0449 | Regression_eta: 0.0254 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0404 | LR: 1.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2282s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank5 [2026-10-30 00:38:07,832] INFO : Step 40600/50000 rank5 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0519 | Regression_eta: 0.0306 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0514 | LR: 1.76e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2967s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank5 [2026-10-30 00:42:54,583] INFO : Step 40700/50000 rank5 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0578 | Regression_eta: 0.0197 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0407 | LR: 1.75e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2546s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank5 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank5 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1075 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0577 | Regression_eta: 0.0269 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0668 | LR: 1.73e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8892s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank5 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank5 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0414 | Regression_eta: 0.0227 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0663 | LR: 1.72e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7092s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank5 [2026-10-30 00:57:38,929] INFO : Step 41000/50000 rank5 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1235 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0371 | Regression_eta: 0.0202 | Regression_pt: 0.0379 | Regression_sin_phi: 0.0364 | LR: 1.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3879s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank5 [2026-10-30 00:57:39,931] INFO : Running validation on rank5 +rank5 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:57:39,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:58:31,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:58:32,295] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 00:58:59,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:59:00,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:59:01,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:59:02,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:59:12,417] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:59:13,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:59:13,669] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 00:59:30,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:59:31,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:59:32,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:59:33,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:59:43,041] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:59:43,889] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 00:59:59,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:00:00,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:00:01,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:00:02,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:00:12,380] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 01:05:10,997] INFO : Step 41100/50000 rank5 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0269 | Classification_binary: 0.0984 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0539 | Regression_eta: 0.0302 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0375 | LR: 1.69e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7671s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank5 [2026-10-30 01:10:09,654] INFO : Step 41200/50000 rank5 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0627 | Regression_eta: 0.0337 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0370 | LR: 1.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6361s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank5 [2026-10-30 01:15:00,325] INFO : Step 41300/50000 rank5 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0442 | Regression_eta: 0.0244 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0538 | LR: 1.66e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2636s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank5 [2026-10-30 01:19:55,522] INFO : Step 41400/50000 rank5 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1215 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0502 | Regression_eta: 0.0336 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0559 | LR: 1.64e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.6709s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank5 [2026-10-30 01:24:55,602] INFO : Step 41500/50000 rank5 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.0944 | Regression_cos_phi: 0.0313 | Regression_energy: 0.0385 | Regression_eta: 0.0215 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0340 | LR: 1.63e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4517s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank5 [2026-10-30 01:29:39,759] INFO : Step 41600/50000 rank5 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0365 | Regression_eta: 0.0324 | Regression_pt: 0.0385 | Regression_sin_phi: 0.0363 | LR: 1.61e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.0140s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank5 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank5 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0304 | Regression_energy: 0.0429 | Regression_eta: 0.0316 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0396 | LR: 1.60e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7035s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank5 [2026-10-30 01:39:10,939] INFO : Step 41800/50000 rank5 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1031 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0504 | Regression_eta: 0.0529 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0514 | LR: 1.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8801s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank5 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank5 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1126 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0333 | Regression_eta: 0.0295 | Regression_pt: 0.0341 | Regression_sin_phi: 0.0468 | LR: 1.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2474s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank5 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank5 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0414 | Regression_eta: 0.0309 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0432 | LR: 1.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2684s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank5 [2026-10-30 01:49:03,869] INFO : Running validation on rank5 +rank5 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 01:49:57,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 01:50:24,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 01:50:26,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 01:50:26,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 01:50:27,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 01:50:37,977] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 01:50:38,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:50:39,486] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 01:50:55,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 01:50:56,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 01:50:57,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 01:50:58,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 01:51:08,221] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:51:10,100] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 01:51:25,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:51:26,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:51:27,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:51:28,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:51:38,236] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank5 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1030 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0557 | Regression_eta: 0.0221 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0474 | LR: 1.54e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0034s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank5 [2026-10-30 02:01:47,477] INFO : Step 42200/50000 rank5 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1057 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0496 | Regression_eta: 0.0240 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0513 | LR: 1.53e-04 | DataLoad Time: 0.0170s | Model Forward Time: 7.2808s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank5 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank5 [2026-10-30 02:02:38,475] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-30 02:06:41,298] INFO : Step 42300/50000 rank5 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0408 | Regression_eta: 0.0245 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0490 | LR: 1.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8851s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank5 [2026-10-30 02:11:46,427] INFO : Step 42400/50000 rank5 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0305 | Classification_binary: 0.1119 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0500 | Regression_eta: 0.0331 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0775 | LR: 1.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.1598s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank5 [2026-10-30 02:16:38,986] INFO : Step 42500/50000 rank5 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0369 | Regression_eta: 0.0339 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0582 | LR: 1.49e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8617s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank5 [2026-10-30 02:21:27,108] INFO : Step 42600/50000 rank5 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0398 | Regression_eta: 0.0287 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0288 | LR: 1.48e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.3531s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank5 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank5 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1123 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0455 | Regression_eta: 0.0374 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0472 | LR: 1.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6234s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank5 [2026-10-30 02:31:38,797] INFO : Step 42800/50000 rank5 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0229 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0325 | Regression_energy: 0.0465 | Regression_eta: 0.0215 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0475 | LR: 1.45e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6419s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank5 [2026-10-30 02:36:26,142] INFO : Step 42900/50000 rank5 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0464 | Regression_eta: 0.0338 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0396 | LR: 1.44e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8546s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank5 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank5 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0493 | Regression_eta: 0.0268 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0569 | LR: 1.43e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5949s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank5 [2026-10-30 02:41:14,879] INFO : Running validation on rank5 +rank5 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 02:42:10,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 02:42:38,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 02:42:39,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 02:42:39,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 02:42:40,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 02:42:51,001] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 02:42:51,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:42:52,441] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 02:43:08,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 02:43:09,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 02:43:10,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 02:43:11,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 02:43:21,454] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:43:23,031] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 02:43:38,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 02:43:39,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 02:43:40,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 02:43:41,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 02:43:51,480] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 02:48:51,376] INFO : Step 43100/50000 rank5 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0319 | Regression_energy: 0.0396 | Regression_eta: 0.0191 | Regression_pt: 0.0413 | Regression_sin_phi: 0.0403 | LR: 1.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9463s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank5 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank5 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0561 | Regression_eta: 0.0333 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0517 | LR: 1.40e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5642s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank5 [2026-10-30 02:58:07,502] INFO : Step 43300/50000 rank5 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0472 | Regression_eta: 0.0319 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0605 | LR: 1.39e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.0507s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank5 [2026-10-30 03:02:42,808] INFO : Step 43400/50000 rank5 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.1099 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0474 | Regression_eta: 0.0186 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0433 | LR: 1.38e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4742s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank5 [2026-10-30 03:07:16,794] INFO : Step 43500/50000 rank5 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0349 | Regression_eta: 0.0274 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0333 | LR: 1.37e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9334s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank5 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank5 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0484 | Regression_eta: 0.0352 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0491 | LR: 1.36e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8420s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank5 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank5 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1218 | Regression_cos_phi: 0.0266 | Regression_energy: 0.0458 | Regression_eta: 0.0303 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0540 | LR: 1.35e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1358s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank5 [2026-10-30 03:21:10,515] INFO : Step 43800/50000 rank5 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0509 | Regression_eta: 0.0447 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0349 | LR: 1.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2541s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank5 [2026-10-30 03:26:01,001] INFO : Step 43900/50000 rank5 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0461 | Regression_eta: 0.0381 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0470 | LR: 1.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9699s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank5 [2026-10-30 03:30:36,934] INFO : Step 44000/50000 rank5 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0451 | Regression_eta: 0.0433 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0541 | LR: 1.32e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8192s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank5 [2026-10-30 03:30:37,938] INFO : Running validation on rank5 +rank5 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 03:31:28,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:31:29,165] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 03:31:57,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 03:31:58,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 03:31:59,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 03:32:00,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 03:32:10,732] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 03:32:27,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 03:32:28,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 03:32:29,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 03:32:30,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 03:32:40,564] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:32:41,628] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 03:32:56,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 03:32:57,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 03:32:58,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 03:32:59,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 03:33:09,420] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 03:38:15,294] INFO : Step 44100/50000 rank5 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0308 | Regression_energy: 0.0514 | Regression_eta: 0.0222 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0459 | LR: 1.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7484s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank5 [2026-10-30 03:43:04,523] INFO : Step 44200/50000 rank5 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0362 | Regression_eta: 0.0240 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0446 | LR: 1.30e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4224s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank5 [2026-10-30 03:47:46,465] INFO : Step 44300/50000 rank5 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1125 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0375 | Regression_eta: 0.0340 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0328 | LR: 1.29e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4337s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank5 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank5 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0456 | Regression_eta: 0.0373 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0456 | LR: 1.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3361s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank5 [2026-10-30 03:57:11,972] INFO : Step 44500/50000 rank5 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0437 | Regression_eta: 0.0303 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0522 | LR: 1.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3175s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank5 [2026-10-30 04:01:48,197] INFO : Step 44600/50000 rank5 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0375 | Regression_eta: 0.0244 | Regression_pt: 0.0388 | Regression_sin_phi: 0.0780 | LR: 1.26e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5951s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank5 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank5 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0431 | Regression_eta: 0.0287 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0420 | LR: 1.25e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.6217s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank5 [2026-10-30 04:11:07,221] INFO : Step 44800/50000 rank5 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1196 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0481 | Regression_eta: 0.0393 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0431 | LR: 1.24e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5876s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank5 [2026-10-30 04:15:54,538] INFO : Step 44900/50000 rank5 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0289 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0309 | Regression_energy: 0.0351 | Regression_eta: 0.0194 | Regression_pt: 0.0367 | Regression_sin_phi: 0.0321 | LR: 1.23e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6722s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank5 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank5 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0317 | Regression_energy: 0.0454 | Regression_eta: 0.0353 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0450 | LR: 1.22e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7895s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank5 [2026-10-30 04:20:37,109] INFO : Running validation on rank5 +rank5 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 04:21:27,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:21:28,557] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 04:21:56,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 04:21:57,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 04:21:58,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 04:21:59,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 04:22:09,932] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:22:11,155] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 04:22:27,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 04:22:28,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 04:22:29,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 04:22:30,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 04:22:40,217] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 04:22:40,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:22:41,055] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 04:22:57,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 04:22:57,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 04:22:58,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 04:22:59,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 04:23:09,612] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 04:28:04,850] INFO : Step 45100/50000 rank5 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0243 | Classification_binary: 0.0962 | Regression_cos_phi: 0.0286 | Regression_energy: 0.0620 | Regression_eta: 0.0416 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0454 | LR: 1.21e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2115s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank5 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank5 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0397 | Regression_eta: 0.0352 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0431 | LR: 1.20e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank5 [2026-10-30 04:37:52,086] INFO : Step 45300/50000 rank5 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0435 | Regression_eta: 0.0348 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0509 | LR: 1.19e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8155s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank5 [2026-10-30 04:42:38,427] INFO : Step 45400/50000 rank5 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1020 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0392 | Regression_eta: 0.0235 | Regression_pt: 0.0403 | Regression_sin_phi: 0.0317 | LR: 1.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4015s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank5 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank5 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0284 | Classification_binary: 0.1061 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0345 | Regression_eta: 0.0196 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0348 | LR: 1.18e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3862s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank5 [2026-10-30 04:52:16,190] INFO : Step 45600/50000 rank5 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0457 | Regression_eta: 0.0319 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0513 | LR: 1.17e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4794s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank5 [2026-10-30 04:57:07,845] INFO : Step 45700/50000 rank5 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1105 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0361 | Regression_eta: 0.0265 | Regression_pt: 0.0371 | Regression_sin_phi: 0.0452 | LR: 1.16e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4878s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank5 [2026-10-30 05:01:56,082] INFO : Step 45800/50000 rank5 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1178 | Regression_cos_phi: 0.0330 | Regression_energy: 0.0617 | Regression_eta: 0.0309 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0582 | LR: 1.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6470s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank5 [2026-10-30 05:06:42,063] INFO : Step 45900/50000 rank5 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0267 | Regression_energy: 0.0435 | Regression_eta: 0.0347 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0290 | LR: 1.15e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2877s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank5 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank5 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0245 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0435 | Regression_eta: 0.0268 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0519 | LR: 1.14e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.4345s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank5 [2026-10-30 05:11:37,038] INFO : Running validation on rank5 +rank5 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 05:12:27,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:12:27,994] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 05:12:53,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 05:12:54,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 05:12:55,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 05:12:56,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 05:13:06,184] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 05:13:23,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 05:13:24,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 05:13:25,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 05:13:26,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 05:13:36,181] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 05:13:36,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:13:37,281] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 05:13:52,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 05:13:53,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 05:13:54,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 05:13:55,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 05:14:05,414] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 05:19:00,537] INFO : Step 46100/50000 rank5 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0439 | Regression_eta: 0.0154 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0321 | LR: 1.13e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5912s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank5 [2026-10-30 05:23:34,447] INFO : Step 46200/50000 rank5 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1063 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0398 | Regression_eta: 0.0222 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0423 | LR: 1.13e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7929s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank5 [2026-10-30 05:28:08,609] INFO : Step 46300/50000 rank5 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0524 | Regression_eta: 0.0355 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0608 | LR: 1.12e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9780s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank5 [2026-10-30 05:32:46,649] INFO : Step 46400/50000 rank5 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1142 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0385 | Regression_eta: 0.0257 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0431 | LR: 1.11e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7894s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank5 [2026-10-30 05:37:21,832] INFO : Step 46500/50000 rank5 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0315 | Regression_eta: 0.0377 | Regression_pt: 0.0319 | Regression_sin_phi: 0.0530 | LR: 1.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8420s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank5 [2026-10-30 05:41:57,344] INFO : Step 46600/50000 rank5 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0538 | Regression_eta: 0.0311 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0552 | LR: 1.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7851s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank5 [2026-10-30 05:46:32,412] INFO : Step 46700/50000 rank5 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0432 | Regression_eta: 0.0343 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0445 | LR: 1.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6551s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank5 [2026-10-30 05:51:05,143] INFO : Step 46800/50000 rank5 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1179 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0383 | Regression_eta: 0.0367 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0520 | LR: 1.09e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7032s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank5 [2026-10-30 05:55:41,598] INFO : Step 46900/50000 rank5 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1235 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0404 | Regression_eta: 0.0307 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0410 | LR: 1.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7532s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank5 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank5 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0404 | Regression_eta: 0.0545 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0474 | LR: 1.08e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8072s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank5 [2026-10-30 06:00:21,750] INFO : Running validation on rank5 +rank5 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 06:00:21,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 06:01:12,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 06:01:37,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 06:01:38,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 06:01:39,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 06:01:40,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 06:01:50,156] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:01:51,923] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 06:02:07,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 06:02:08,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 06:02:09,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 06:02:10,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 06:02:20,736] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:02:21,871] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 06:02:37,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 06:02:38,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 06:02:39,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 06:02:40,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 06:02:50,045] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 06:07:51,195] INFO : Step 47100/50000 rank5 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0442 | Regression_eta: 0.0443 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0785 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6258s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank5 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank5 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1035 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0364 | Regression_eta: 0.0212 | Regression_pt: 0.0382 | Regression_sin_phi: 0.0326 | LR: 1.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9891s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank5 [2026-10-30 06:17:04,150] INFO : Step 47300/50000 rank5 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0491 | Regression_eta: 0.0434 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0852 | LR: 1.06e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0309s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank5 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank5 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0459 | Regression_eta: 0.0335 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0621 | LR: 1.06e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4519s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank5 [2026-10-30 06:26:15,800] INFO : Step 47500/50000 rank5 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1121 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0556 | Regression_eta: 0.0266 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0507 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6765s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank5 [2026-10-30 06:30:50,119] INFO : Step 47600/50000 rank5 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0358 | Regression_eta: 0.0448 | Regression_pt: 0.0374 | Regression_sin_phi: 0.0591 | LR: 1.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.9498s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank5 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank5 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0438 | Regression_eta: 0.0441 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0457 | LR: 1.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9097s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank5 [2026-10-30 06:40:00,408] INFO : Step 47800/50000 rank5 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0278 | Classification_binary: 0.0919 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0357 | Regression_eta: 0.0231 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0392 | LR: 1.04e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7799s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank5 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank5 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0335 | Regression_energy: 0.0326 | Regression_eta: 0.0202 | Regression_pt: 0.0333 | Regression_sin_phi: 0.0362 | LR: 1.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8492s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank5 [2026-10-30 06:49:09,637] INFO : Step 48000/50000 rank5 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0417 | Regression_eta: 0.0319 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0378 | LR: 1.04e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8821s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank5 [2026-10-30 06:49:10,727] INFO : Running validation on rank5 +rank5 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 06:50:01,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 06:50:27,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 06:50:28,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 06:50:29,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 06:50:30,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 06:50:40,043] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:50:41,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:50:41,780] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 06:50:57,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 06:50:58,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 06:50:59,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 06:51:00,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 06:51:10,599] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:51:11,431] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 06:51:27,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 06:51:28,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 06:51:28,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 06:51:29,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 06:51:39,797] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank5 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0469 | Regression_eta: 0.0258 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0365 | LR: 1.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6276s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank5 [2026-10-30 07:01:16,422] INFO : Step 48200/50000 rank5 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0229 | Classification_binary: 0.0867 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0399 | Regression_eta: 0.0188 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0269 | LR: 1.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3158s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank5 [2026-10-30 07:05:53,465] INFO : Step 48300/50000 rank5 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0492 | Regression_eta: 0.0283 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0399 | LR: 1.03e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1174s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank5 [2026-10-30 07:10:30,727] INFO : Step 48400/50000 rank5 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1215 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0346 | Regression_eta: 0.0238 | Regression_pt: 0.0343 | Regression_sin_phi: 0.0414 | LR: 1.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8699s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank5 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank5 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1127 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0457 | Regression_eta: 0.0332 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0472 | LR: 1.02e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.9610s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank5 [2026-10-30 07:19:52,538] INFO : Step 48600/50000 rank5 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0514 | Regression_eta: 0.0382 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0756 | LR: 1.02e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8074s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank5 [2026-10-30 07:24:30,058] INFO : Step 48700/50000 rank5 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0429 | Regression_eta: 0.0377 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0414 | LR: 1.02e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9972s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank5 [2026-10-30 07:29:10,217] INFO : Step 48800/50000 rank5 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0470 | Regression_eta: 0.0274 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0406 | LR: 1.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9222s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank5 [2026-10-30 07:33:48,297] INFO : Step 48900/50000 rank5 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0412 | Regression_eta: 0.0450 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0472 | LR: 1.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7896s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank5 [2026-10-30 07:38:22,688] INFO : Step 49000/50000 rank5 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0412 | Regression_eta: 0.0260 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0391 | LR: 1.01e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8896s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank5 [2026-10-30 07:38:23,699] INFO : Running validation on rank5 +rank5 [2026-10-30 07:38:23,700] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 07:38:23,700] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:39:14,632] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 07:39:39,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 07:39:40,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 07:39:41,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 07:39:42,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 07:39:52,459] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 07:39:53,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:39:53,776] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 07:40:09,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 07:40:10,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 07:40:11,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 07:40:12,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 07:40:22,249] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:40:24,011] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 07:40:39,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 07:40:40,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 07:40:41,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 07:40:42,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 07:40:52,349] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 07:45:46,211] INFO : Step 49100/50000 rank5 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0390 | Regression_eta: 0.0320 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0490 | LR: 1.01e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6576s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank5 [2026-10-30 07:50:30,115] INFO : Step 49200/50000 rank5 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0363 | Regression_energy: 0.0350 | Regression_eta: 0.0237 | Regression_pt: 0.0367 | Regression_sin_phi: 0.0527 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5158s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank5 [2026-10-30 07:55:07,371] INFO : Step 49300/50000 rank5 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0413 | Regression_eta: 0.0341 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0615 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8678s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank5 [2026-10-30 07:59:43,351] INFO : Step 49400/50000 rank5 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1263 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0311 | Regression_eta: 0.0252 | Regression_pt: 0.0326 | Regression_sin_phi: 0.0525 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8902s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank5 [2026-10-30 08:04:24,758] INFO : Step 49500/50000 rank5 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0268 | Classification_binary: 0.0768 | Regression_cos_phi: 0.0321 | Regression_energy: 0.0435 | Regression_eta: 0.0216 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0524 | LR: 1.00e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3357s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank5 [2026-10-30 08:08:58,924] INFO : Step 49600/50000 rank5 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0461 | Regression_eta: 0.0191 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0457 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8757s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank5 [2026-10-30 08:13:36,933] INFO : Step 49700/50000 rank5 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0333 | Regression_eta: 0.0261 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0359 | LR: 1.00e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7059s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank5 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank5 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.0847 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0435 | Regression_eta: 0.0475 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0318 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5276s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank5 [2026-10-30 08:22:53,417] INFO : Step 49900/50000 rank5 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0409 | Regression_eta: 0.0409 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0670 | LR: 1.00e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2021s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank5 [2026-10-30 08:27:37,252] INFO : Step 50000/50000 rank5 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0275 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0388 | Regression_eta: 0.0266 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0375 | LR: 1.00e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.3493s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank5 [2026-10-30 08:27:38,187] INFO : Running validation on rank5 +rank5 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 08:28:29,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 08:28:29,585] INFO : Running predictions on clic_edm_ttbar_hits +rank5 [2026-10-30 08:29:00,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 08:29:01,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 08:29:02,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 08:29:03,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 08:29:13,128] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 08:29:13,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 08:29:14,403] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank5 [2026-10-30 08:29:30,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 08:29:32,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 08:29:33,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 08:29:35,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 08:29:45,540] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 08:29:46,238] INFO : Running predictions on clic_edm_qq_hits +rank5 [2026-10-30 08:30:01,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 08:30:02,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 08:30:03,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 08:30:04,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 08:30:14,273] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 08:30:15,192] INFO : Training completed. Total time on device 5: 1059.924min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.6 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..96279b50143d18a12f263e9fd00b650982bee200 --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.6 @@ -0,0 +1,2654 @@ +rank6 [2026-10-28 11:12:32,882] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-28 11:12:32,883] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank6 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-28 11:12:57,192] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank6 [2026-10-28 11:12:57,209] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-28 11:12:57,209] INFO : Initializing backbone layer 0 +rank6 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank6 [2026-10-28 11:12:57,253] INFO : Initializing backbone layer 2 +rank6 [2026-10-28 11:12:57,262] INFO : Initializing backbone layer 3 +rank6 [2026-10-28 11:12:57,270] INFO : Initializing backbone layer 4 +rank6 [2026-10-28 11:12:57,279] INFO : Initializing backbone layer 5 +rank6 [2026-10-28 11:12:57,287] INFO : Backbone initialization took 0.08s +rank6 [2026-10-28 11:12:57,288] INFO : backbone parameters: 7901376 +rank6 [2026-10-28 11:12:57,288] INFO : Initializing output DNNs +rank6 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank6 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank6 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank6 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank6 [2026-10-28 11:12:57,328] INFO : classification_norm parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : regression_norm parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : classification_query parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : regression_query parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : classification_readout parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : regression_readout parameters: 0 +rank6 [2026-10-28 11:12:57,328] INFO : output_mode=set +rank6 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank6 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank6 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank6 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank6 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=6 +rank6 [2026-10-28 11:12:57,356] INFO : Moved model to device rank=6 +rank6 [2026-10-28 11:12:57,357] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-28 11:12:57,358] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-28 11:12:57,693] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-28 11:12:57,854] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank6 [2026-10-28 11:12:57,854] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-28 11:12:57,855] INFO : Trainable parameters: 12786964 +rank6 [2026-10-28 11:12:57,855] INFO : Non-trainable parameters: 23268 +rank6 [2026-10-28 11:12:57,855] INFO : Total parameters: 12810232 +rank6 [2026-10-28 11:12:57,859] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-28 11:12:57,859] INFO : Getting dataloaders +rank6 [2026-10-28 11:12:57,859] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:03,702] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:04,778] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,003] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-28 11:13:06,003] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-28 11:13:06,004] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,276] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,543] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,801] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-28 11:13:06,801] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-28 11:13:20,885] INFO : Got dataloaders +rank6 [2026-10-28 11:13:20,885] INFO : loader split=train rank=6 len=21108 +rank6 [2026-10-28 11:13:20,885] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-28 11:13:20,885] INFO : Starting training from step 1 to 50000 on rank 6 of 8 +rank6 [2026-10-28 11:13:20,885] INFO : Creating train_iterator +rank6 [2026-10-28 11:13:20,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-28 11:13:35,599] INFO : Created train_iterator +rank6 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank6 [2026-10-28 11:18:16,511] INFO : Step 100/50000 rank6 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6791 | Classification_binary: 0.4973 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8269 | Regression_eta: 0.0109 | Regression_pt: 0.7970 | Regression_sin_phi: 0.0057 | LR: 1.00e-03 | DataLoad Time: 0.0152s | Model Forward Time: 3.6179s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank6 [2026-10-28 11:22:47,962] INFO : Step 200/50000 rank6 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3655 | Classification_binary: 0.5050 | Regression_cos_phi: 0.4289 | Regression_energy: 0.3597 | Regression_eta: 0.3268 | Regression_pt: 0.3625 | Regression_sin_phi: 0.4178 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.4382s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank6 [2026-10-28 11:27:17,892] INFO : Step 300/50000 rank6 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2918 | Classification_binary: 0.4837 | Regression_cos_phi: 0.3397 | Regression_energy: 0.2840 | Regression_eta: 0.2745 | Regression_pt: 0.2966 | Regression_sin_phi: 0.2866 | LR: 1.00e-03 | DataLoad Time: 0.0166s | Model Forward Time: 4.3678s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank6 [2026-10-28 11:31:48,895] INFO : Step 400/50000 rank6 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2553 | Classification_binary: 0.4720 | Regression_cos_phi: 0.3204 | Regression_energy: 0.3524 | Regression_eta: 0.2761 | Regression_pt: 0.3426 | Regression_sin_phi: 0.2928 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.2447s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank6 [2026-10-28 11:36:20,164] INFO : Step 500/50000 rank6 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2579 | Classification_binary: 0.5025 | Regression_cos_phi: 0.2581 | Regression_energy: 0.2911 | Regression_eta: 0.3079 | Regression_pt: 0.3069 | Regression_sin_phi: 0.3319 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 4.4682s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank6 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank6 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2631 | Classification_binary: 0.4356 | Regression_cos_phi: 0.2297 | Regression_energy: 0.3273 | Regression_eta: 0.1794 | Regression_pt: 0.3482 | Regression_sin_phi: 0.2945 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.5139s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank6 [2026-10-28 11:45:32,573] INFO : Step 700/50000 rank6 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2149 | Classification_binary: 0.4118 | Regression_cos_phi: 0.2759 | Regression_energy: 0.2841 | Regression_eta: 0.1699 | Regression_pt: 0.2968 | Regression_sin_phi: 0.2507 | LR: 1.00e-03 | DataLoad Time: 0.0178s | Model Forward Time: 3.7400s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank6 [2026-10-28 11:50:15,216] INFO : Step 800/50000 rank6 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2946 | Classification_binary: 0.4569 | Regression_cos_phi: 0.3552 | Regression_energy: 0.3030 | Regression_eta: 0.2470 | Regression_pt: 0.3147 | Regression_sin_phi: 0.3345 | LR: 9.99e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2817s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank6 [2026-10-28 11:54:58,536] INFO : Step 900/50000 rank6 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2417 | Classification_binary: 0.4769 | Regression_cos_phi: 0.2639 | Regression_energy: 0.2515 | Regression_eta: 0.1930 | Regression_pt: 0.2610 | Regression_sin_phi: 0.2791 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4558s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank6 [2026-10-28 11:59:55,247] INFO : Step 1000/50000 rank6 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1899 | Classification_binary: 0.3740 | Regression_cos_phi: 0.1470 | Regression_energy: 0.2778 | Regression_eta: 0.1099 | Regression_pt: 0.2821 | Regression_sin_phi: 0.1643 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.7553s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank6 [2026-10-28 11:59:56,332] INFO : Running validation on rank6 +rank6 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 12:00:48,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:00:49,301] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 12:01:16,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:01:17,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:01:18,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:01:19,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:01:29,944] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:01:30,839] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 12:01:46,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:01:47,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:01:48,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:01:49,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:01:59,698] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:02:00,465] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:02:01,007] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 12:02:16,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:02:17,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:02:18,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:02:19,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:02:29,098] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 12:07:33,552] INFO : Step 1100/50000 rank6 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2130 | Classification_binary: 0.4203 | Regression_cos_phi: 0.1676 | Regression_energy: 0.2737 | Regression_eta: 0.1605 | Regression_pt: 0.2804 | Regression_sin_phi: 0.1884 | LR: 9.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8146s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank6 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank6 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1975 | Classification_binary: 0.4758 | Regression_cos_phi: 0.2108 | Regression_energy: 0.2474 | Regression_eta: 0.1866 | Regression_pt: 0.2594 | Regression_sin_phi: 0.2318 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4371s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank6 [2026-10-28 12:17:07,815] INFO : Step 1300/50000 rank6 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1607 | Classification_binary: 0.4353 | Regression_cos_phi: 0.2360 | Regression_energy: 0.2219 | Regression_eta: 0.1891 | Regression_pt: 0.2363 | Regression_sin_phi: 0.1862 | LR: 9.98e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2262s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank6 [2026-10-28 12:21:55,310] INFO : Step 1400/50000 rank6 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2318 | Classification_binary: 0.3878 | Regression_cos_phi: 0.2271 | Regression_energy: 0.2500 | Regression_eta: 0.2268 | Regression_pt: 0.2601 | Regression_sin_phi: 0.2562 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5558s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank6 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank6 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1988 | Classification_binary: 0.4372 | Regression_cos_phi: 0.2144 | Regression_energy: 0.2230 | Regression_eta: 0.1830 | Regression_pt: 0.2341 | Regression_sin_phi: 0.2179 | LR: 9.98e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9973s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank6 [2026-10-28 12:31:13,938] INFO : Step 1600/50000 rank6 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 0.4478 | Regression_cos_phi: 0.2175 | Regression_energy: 0.2229 | Regression_eta: 0.1824 | Regression_pt: 0.2299 | Regression_sin_phi: 0.2000 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8067s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank6 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank6 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1681 | Classification_binary: 0.3630 | Regression_cos_phi: 0.1654 | Regression_energy: 0.2439 | Regression_eta: 0.1146 | Regression_pt: 0.2487 | Regression_sin_phi: 0.1189 | LR: 9.97e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6764s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank6 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank6 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1728 | Classification_binary: 0.3889 | Regression_cos_phi: 0.1590 | Regression_energy: 0.2062 | Regression_eta: 0.1310 | Regression_pt: 0.2149 | Regression_sin_phi: 0.2110 | LR: 9.97e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6768s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank6 [2026-10-28 12:45:37,437] INFO : Step 1900/50000 rank6 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2267 | Classification_binary: 0.4948 | Regression_cos_phi: 0.2303 | Regression_energy: 0.2332 | Regression_eta: 0.1912 | Regression_pt: 0.2465 | Regression_sin_phi: 0.2559 | LR: 9.97e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4296s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank6 [2026-10-28 12:50:22,598] INFO : Step 2000/50000 rank6 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1749 | Classification_binary: 0.3526 | Regression_cos_phi: 0.1941 | Regression_energy: 0.2119 | Regression_eta: 0.1457 | Regression_pt: 0.2141 | Regression_sin_phi: 0.1861 | LR: 9.96e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.9891s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank6 [2026-10-28 12:50:23,688] INFO : Running validation on rank6 +rank6 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 12:51:19,821] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:51:20,337] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 12:51:47,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:51:48,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:51:50,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:51:51,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:52:01,172] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 12:52:18,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:52:19,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:52:20,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:52:21,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:52:31,298] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:52:31,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:52:32,091] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 12:52:48,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:52:49,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:52:50,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:52:51,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:53:01,611] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank6 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2178 | Classification_binary: 0.3742 | Regression_cos_phi: 0.2248 | Regression_energy: 0.2236 | Regression_eta: 0.1684 | Regression_pt: 0.2284 | Regression_sin_phi: 0.2525 | LR: 9.96e-04 | DataLoad Time: 0.0111s | Model Forward Time: 4.6419s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank6 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank6 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1731 | Classification_binary: 0.4348 | Regression_cos_phi: 0.2359 | Regression_energy: 0.2677 | Regression_eta: 0.1795 | Regression_pt: 0.2793 | Regression_sin_phi: 0.1909 | LR: 9.96e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.4345s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank6 [2026-10-28 13:08:12,324] INFO : Step 2300/50000 rank6 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1148 | Classification_binary: 0.3612 | Regression_cos_phi: 0.1481 | Regression_energy: 0.2202 | Regression_eta: 0.1377 | Regression_pt: 0.2263 | Regression_sin_phi: 0.1716 | LR: 9.95e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1080s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank6 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank6 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1654 | Classification_binary: 0.3205 | Regression_cos_phi: 0.2311 | Regression_energy: 0.2324 | Regression_eta: 0.1497 | Regression_pt: 0.2384 | Regression_sin_phi: 0.2081 | LR: 9.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2190s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank6 [2026-10-28 13:18:06,904] INFO : Step 2500/50000 rank6 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1913 | Classification_binary: 0.4008 | Regression_cos_phi: 0.1951 | Regression_energy: 0.2361 | Regression_eta: 0.2131 | Regression_pt: 0.2497 | Regression_sin_phi: 0.2270 | LR: 9.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.0384s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank6 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank6 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1406 | Classification_binary: 0.4121 | Regression_cos_phi: 0.2032 | Regression_energy: 0.2352 | Regression_eta: 0.1726 | Regression_pt: 0.2443 | Regression_sin_phi: 0.2098 | LR: 9.94e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3780s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank6 [2026-10-28 13:27:39,806] INFO : Step 2700/50000 rank6 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1289 | Classification_binary: 0.2978 | Regression_cos_phi: 0.2764 | Regression_energy: 0.2197 | Regression_eta: 0.1789 | Regression_pt: 0.2293 | Regression_sin_phi: 0.1936 | LR: 9.94e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5228s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank6 [2026-10-28 13:32:27,615] INFO : Step 2800/50000 rank6 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1632 | Classification_binary: 0.3569 | Regression_cos_phi: 0.2375 | Regression_energy: 0.2083 | Regression_eta: 0.1403 | Regression_pt: 0.2152 | Regression_sin_phi: 0.2084 | LR: 9.93e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2927s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank6 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank6 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1979 | Classification_binary: 0.4026 | Regression_cos_phi: 0.2349 | Regression_energy: 0.2525 | Regression_eta: 0.2101 | Regression_pt: 0.2648 | Regression_sin_phi: 0.2568 | LR: 9.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5096s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank6 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank6 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1736 | Classification_binary: 0.3863 | Regression_cos_phi: 0.2180 | Regression_energy: 0.2369 | Regression_eta: 0.1907 | Regression_pt: 0.2424 | Regression_sin_phi: 0.2731 | LR: 9.92e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5892s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank6 [2026-10-28 13:42:04,678] INFO : Running validation on rank6 +rank6 [2026-10-28 13:42:04,679] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:42:04,679] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:42:57,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:42:58,469] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 13:43:25,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:43:26,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:43:27,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:43:28,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:43:38,317] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:43:39,344] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 13:43:54,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:43:56,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:43:56,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:43:57,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:44:07,986] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:44:09,100] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 13:44:24,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:44:25,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:44:26,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:44:26,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:44:36,964] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 13:49:29,793] INFO : Step 3100/50000 rank6 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1722 | Classification_binary: 0.3132 | Regression_cos_phi: 0.2375 | Regression_energy: 0.2054 | Regression_eta: 0.1411 | Regression_pt: 0.2162 | Regression_sin_phi: 0.1888 | LR: 9.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8452s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank6 [2026-10-28 13:54:22,657] INFO : Step 3200/50000 rank6 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1681 | Classification_binary: 0.3866 | Regression_cos_phi: 0.1803 | Regression_energy: 0.2026 | Regression_eta: 0.1374 | Regression_pt: 0.2096 | Regression_sin_phi: 0.2058 | LR: 9.91e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0536s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank6 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank6 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1675 | Classification_binary: 0.3524 | Regression_cos_phi: 0.2112 | Regression_energy: 0.1954 | Regression_eta: 0.1535 | Regression_pt: 0.2053 | Regression_sin_phi: 0.2144 | LR: 9.90e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7399s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank6 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank6 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1330 | Classification_binary: 0.3294 | Regression_cos_phi: 0.1996 | Regression_energy: 0.1927 | Regression_eta: 0.1253 | Regression_pt: 0.2003 | Regression_sin_phi: 0.1968 | LR: 9.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9450s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank6 [2026-10-28 14:08:16,394] INFO : Step 3500/50000 rank6 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1544 | Classification_binary: 0.3660 | Regression_cos_phi: 0.2362 | Regression_energy: 0.2020 | Regression_eta: 0.1638 | Regression_pt: 0.2141 | Regression_sin_phi: 0.1586 | LR: 9.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7451s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank6 [2026-10-28 14:13:02,152] INFO : Step 3600/50000 rank6 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.4176 | Regression_cos_phi: 0.2252 | Regression_energy: 0.2120 | Regression_eta: 0.1278 | Regression_pt: 0.2174 | Regression_sin_phi: 0.1782 | LR: 9.89e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8792s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank6 [2026-10-28 14:17:42,402] INFO : Step 3700/50000 rank6 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.2977 | Regression_cos_phi: 0.2036 | Regression_energy: 0.1770 | Regression_eta: 0.1632 | Regression_pt: 0.1840 | Regression_sin_phi: 0.2003 | LR: 9.88e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.2551s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank6 [2026-10-28 14:22:44,342] INFO : Step 3800/50000 rank6 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1261 | Classification_binary: 0.2906 | Regression_cos_phi: 0.1566 | Regression_energy: 0.2256 | Regression_eta: 0.1627 | Regression_pt: 0.2367 | Regression_sin_phi: 0.2063 | LR: 9.87e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.1810s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank6 [2026-10-28 14:27:33,637] INFO : Step 3900/50000 rank6 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1472 | Classification_binary: 0.4091 | Regression_cos_phi: 0.2195 | Regression_energy: 0.2008 | Regression_eta: 0.1526 | Regression_pt: 0.2095 | Regression_sin_phi: 0.2204 | LR: 9.87e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.8210s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank6 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank6 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.3172 | Regression_cos_phi: 0.1990 | Regression_energy: 0.1942 | Regression_eta: 0.1363 | Regression_pt: 0.1981 | Regression_sin_phi: 0.2221 | LR: 9.86e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5861s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank6 [2026-10-28 14:32:39,608] INFO : Running validation on rank6 +rank6 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 14:33:34,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:33:34,721] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 14:34:03,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 14:34:08,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 14:34:10,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 14:34:11,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 14:34:21,381] INFO : Time taken to make predictions on device 6 is: 0.35 min +rank6 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:34:24,285] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 14:34:40,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 14:34:43,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 14:34:44,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 14:34:45,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 14:34:55,201] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:34:56,346] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 14:35:12,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 14:35:12,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 14:35:13,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 14:35:14,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 14:35:24,662] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank6 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1659 | Classification_binary: 0.3592 | Regression_cos_phi: 0.2358 | Regression_energy: 0.2635 | Regression_eta: 0.1738 | Regression_pt: 0.2658 | Regression_sin_phi: 0.2097 | LR: 9.85e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8518s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank6 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank6 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1697 | Classification_binary: 0.3817 | Regression_cos_phi: 0.2208 | Regression_energy: 0.2089 | Regression_eta: 0.2136 | Regression_pt: 0.2153 | Regression_sin_phi: 0.2209 | LR: 9.84e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5083s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank6 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank6 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.3313 | Regression_cos_phi: 0.1612 | Regression_energy: 0.1829 | Regression_eta: 0.1439 | Regression_pt: 0.1883 | Regression_sin_phi: 0.1280 | LR: 9.84e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4590s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank6 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank6 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1080 | Classification_binary: 0.2715 | Regression_cos_phi: 0.1036 | Regression_energy: 0.2059 | Regression_eta: 0.0974 | Regression_pt: 0.2144 | Regression_sin_phi: 0.1550 | LR: 9.83e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1740s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank6 [2026-10-28 14:59:28,687] INFO : Step 4500/50000 rank6 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.3687 | Regression_cos_phi: 0.2806 | Regression_energy: 0.1968 | Regression_eta: 0.1725 | Regression_pt: 0.1996 | Regression_sin_phi: 0.2949 | LR: 9.82e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2521s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank6 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank6 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1472 | Classification_binary: 0.4281 | Regression_cos_phi: 0.1853 | Regression_energy: 0.1881 | Regression_eta: 0.1722 | Regression_pt: 0.1956 | Regression_sin_phi: 0.2177 | LR: 9.81e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7021s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank6 [2026-10-28 15:09:18,561] INFO : Step 4700/50000 rank6 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1395 | Classification_binary: 0.3129 | Regression_cos_phi: 0.1731 | Regression_energy: 0.1740 | Regression_eta: 0.1565 | Regression_pt: 0.1773 | Regression_sin_phi: 0.1871 | LR: 9.81e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7058s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank6 [2026-10-28 15:14:30,754] INFO : Step 4800/50000 rank6 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1284 | Classification_binary: 0.3163 | Regression_cos_phi: 0.1864 | Regression_energy: 0.1937 | Regression_eta: 0.1319 | Regression_pt: 0.2015 | Regression_sin_phi: 0.2000 | LR: 9.80e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4413s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank6 [2026-10-28 15:19:27,571] INFO : Step 4900/50000 rank6 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1770 | Classification_binary: 0.3798 | Regression_cos_phi: 0.2568 | Regression_energy: 0.1774 | Regression_eta: 0.1579 | Regression_pt: 0.1854 | Regression_sin_phi: 0.2315 | LR: 9.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5350s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank6 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank6 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1433 | Classification_binary: 0.3441 | Regression_cos_phi: 0.1708 | Regression_energy: 0.1777 | Regression_eta: 0.1487 | Regression_pt: 0.1868 | Regression_sin_phi: 0.2245 | LR: 9.78e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4822s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank6 [2026-10-28 15:24:24,982] INFO : Running validation on rank6 +rank6 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 15:25:19,075] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:25:19,570] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 15:25:44,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 15:25:46,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 15:25:47,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 15:25:48,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 15:25:58,107] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:25:59,250] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 15:26:14,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 15:26:15,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 15:26:16,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 15:26:17,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 15:26:27,913] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:26:28,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 15:26:45,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 15:26:46,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 15:26:46,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 15:26:47,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 15:26:57,807] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 15:32:01,930] INFO : Step 5100/50000 rank6 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.3075 | Regression_cos_phi: 0.2133 | Regression_energy: 0.1792 | Regression_eta: 0.1284 | Regression_pt: 0.1859 | Regression_sin_phi: 0.1635 | LR: 9.77e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.6418s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank6 [2026-10-28 15:36:53,511] INFO : Step 5200/50000 rank6 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1360 | Classification_binary: 0.3586 | Regression_cos_phi: 0.2218 | Regression_energy: 0.1757 | Regression_eta: 0.1493 | Regression_pt: 0.1848 | Regression_sin_phi: 0.1904 | LR: 9.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6383s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank6 [2026-10-28 15:41:36,526] INFO : Step 5300/50000 rank6 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1317 | Classification_binary: 0.3435 | Regression_cos_phi: 0.2029 | Regression_energy: 0.1939 | Regression_eta: 0.1455 | Regression_pt: 0.2023 | Regression_sin_phi: 0.1848 | LR: 9.75e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9251s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank6 [2026-10-28 15:46:33,435] INFO : Step 5400/50000 rank6 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1139 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1750 | Regression_energy: 0.1615 | Regression_eta: 0.1173 | Regression_pt: 0.1597 | Regression_sin_phi: 0.1178 | LR: 9.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4639s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank6 [2026-10-28 15:51:26,877] INFO : Step 5500/50000 rank6 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2685 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1950 | Regression_eta: 0.1006 | Regression_pt: 0.2047 | Regression_sin_phi: 0.1293 | LR: 9.73e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.5335s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank6 [2026-10-28 15:56:25,712] INFO : Step 5600/50000 rank6 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1469 | Classification_binary: 0.3985 | Regression_cos_phi: 0.2034 | Regression_energy: 0.1693 | Regression_eta: 0.1415 | Regression_pt: 0.1751 | Regression_sin_phi: 0.1873 | LR: 9.72e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.3184s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank6 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank6 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1043 | Classification_binary: 0.2951 | Regression_cos_phi: 0.1828 | Regression_energy: 0.1525 | Regression_eta: 0.1278 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1713 | LR: 9.71e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5520s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank6 [2026-10-28 16:06:09,458] INFO : Step 5800/50000 rank6 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2703 | Regression_cos_phi: 0.0949 | Regression_energy: 0.1750 | Regression_eta: 0.0896 | Regression_pt: 0.1799 | Regression_sin_phi: 0.1369 | LR: 9.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2266s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank6 [2026-10-28 16:11:19,900] INFO : Step 5900/50000 rank6 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1522 | Classification_binary: 0.3168 | Regression_cos_phi: 0.1697 | Regression_energy: 0.1804 | Regression_eta: 0.1592 | Regression_pt: 0.1885 | Regression_sin_phi: 0.2181 | LR: 9.69e-04 | DataLoad Time: 0.0155s | Model Forward Time: 7.1482s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank6 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank6 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1382 | Classification_binary: 0.3379 | Regression_cos_phi: 0.1673 | Regression_energy: 0.1630 | Regression_eta: 0.1603 | Regression_pt: 0.1678 | Regression_sin_phi: 0.1616 | LR: 9.68e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4965s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank6 [2026-10-28 16:16:41,624] INFO : Running validation on rank6 +rank6 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:17:32,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:17:33,086] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 16:17:59,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:18:00,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:18:01,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:18:02,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:18:12,620] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:18:13,875] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 16:18:29,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:18:30,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:18:31,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:18:32,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:18:42,919] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:18:43,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:18:43,953] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 16:18:59,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:19:00,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:19:02,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:19:03,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:19:13,741] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank6 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1115 | Classification_binary: 0.2419 | Regression_cos_phi: 0.1183 | Regression_energy: 0.1335 | Regression_eta: 0.1601 | Regression_pt: 0.1387 | Regression_sin_phi: 0.1467 | LR: 9.67e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9548s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank6 [2026-10-28 16:29:26,898] INFO : Step 6200/50000 rank6 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1676 | Classification_binary: 0.3360 | Regression_cos_phi: 0.1884 | Regression_energy: 0.1687 | Regression_eta: 0.1638 | Regression_pt: 0.1772 | Regression_sin_phi: 0.2107 | LR: 9.66e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0911s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank6 [2026-10-28 16:34:17,121] INFO : Step 6300/50000 rank6 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1584 | Classification_binary: 0.3882 | Regression_cos_phi: 0.1699 | Regression_energy: 0.1589 | Regression_eta: 0.1438 | Regression_pt: 0.1657 | Regression_sin_phi: 0.1968 | LR: 9.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0312s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank6 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank6 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1040 | Classification_binary: 0.2698 | Regression_cos_phi: 0.1549 | Regression_energy: 0.1727 | Regression_eta: 0.0976 | Regression_pt: 0.1768 | Regression_sin_phi: 0.1156 | LR: 9.64e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7764s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank6 [2026-10-28 16:43:58,054] INFO : Step 6500/50000 rank6 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1323 | Classification_binary: 0.3252 | Regression_cos_phi: 0.1906 | Regression_energy: 0.1767 | Regression_eta: 0.1372 | Regression_pt: 0.1860 | Regression_sin_phi: 0.2230 | LR: 9.63e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.4995s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank6 [2026-10-28 16:48:52,484] INFO : Step 6600/50000 rank6 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1259 | Classification_binary: 0.3581 | Regression_cos_phi: 0.1770 | Regression_energy: 0.1455 | Regression_eta: 0.1202 | Regression_pt: 0.1507 | Regression_sin_phi: 0.1555 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8078s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank6 [2026-10-28 16:53:47,414] INFO : Step 6700/50000 rank6 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1181 | Classification_binary: 0.2970 | Regression_cos_phi: 0.1681 | Regression_energy: 0.1485 | Regression_eta: 0.1360 | Regression_pt: 0.1530 | Regression_sin_phi: 0.1385 | LR: 9.61e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.3357s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank6 [2026-10-28 16:58:34,482] INFO : Step 6800/50000 rank6 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1057 | Classification_binary: 0.2667 | Regression_cos_phi: 0.1280 | Regression_energy: 0.1601 | Regression_eta: 0.1087 | Regression_pt: 0.1638 | Regression_sin_phi: 0.1332 | LR: 9.60e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.4637s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank6 [2026-10-28 17:03:31,774] INFO : Step 6900/50000 rank6 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1492 | Classification_binary: 0.3537 | Regression_cos_phi: 0.1983 | Regression_energy: 0.1586 | Regression_eta: 0.1584 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1382 | LR: 9.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6896s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank6 [2026-10-28 17:08:29,915] INFO : Step 7000/50000 rank6 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.3761 | Regression_cos_phi: 0.3035 | Regression_energy: 0.1447 | Regression_eta: 0.1714 | Regression_pt: 0.1467 | Regression_sin_phi: 0.2827 | LR: 9.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9227s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank6 [2026-10-28 17:08:30,791] INFO : Running validation on rank6 +rank6 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 17:09:23,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:09:24,267] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 17:09:52,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 17:09:53,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 17:09:55,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 17:09:57,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 17:10:07,126] INFO : Time taken to make predictions on device 6 is: 0.30 min +rank6 [2026-10-28 17:10:07,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:10:08,262] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 17:10:23,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 17:10:24,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 17:10:25,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 17:10:26,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 17:10:36,858] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:10:37,866] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:10:38,448] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 17:10:54,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 17:10:54,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 17:10:55,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 17:10:56,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 17:11:06,646] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 17:16:27,595] INFO : Step 7100/50000 rank6 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2483 | Regression_cos_phi: 0.1158 | Regression_energy: 0.1627 | Regression_eta: 0.1017 | Regression_pt: 0.1666 | Regression_sin_phi: 0.1344 | LR: 9.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1479s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank6 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank6 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.3034 | Regression_cos_phi: 0.1604 | Regression_energy: 0.1476 | Regression_eta: 0.1122 | Regression_pt: 0.1525 | Regression_sin_phi: 0.1435 | LR: 9.55e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3462s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank6 [2026-10-28 17:26:34,943] INFO : Step 7300/50000 rank6 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.3499 | Regression_cos_phi: 0.1539 | Regression_energy: 0.1527 | Regression_eta: 0.1230 | Regression_pt: 0.1581 | Regression_sin_phi: 0.1918 | LR: 9.53e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4010s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank6 [2026-10-28 17:31:34,684] INFO : Step 7400/50000 rank6 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0953 | Classification_binary: 0.2707 | Regression_cos_phi: 0.0992 | Regression_energy: 0.1675 | Regression_eta: 0.0886 | Regression_pt: 0.1741 | Regression_sin_phi: 0.1118 | LR: 9.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank6 [2026-10-28 17:36:30,884] INFO : Step 7500/50000 rank6 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.2361 | Regression_cos_phi: 0.1638 | Regression_energy: 0.1371 | Regression_eta: 0.1074 | Regression_pt: 0.1439 | Regression_sin_phi: 0.1546 | LR: 9.51e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6592s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank6 [2026-10-28 17:41:09,113] INFO : Step 7600/50000 rank6 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1371 | Classification_binary: 0.3158 | Regression_cos_phi: 0.1763 | Regression_energy: 0.1448 | Regression_eta: 0.1294 | Regression_pt: 0.1491 | Regression_sin_phi: 0.1836 | LR: 9.50e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7898s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank6 [2026-10-28 17:46:03,243] INFO : Step 7700/50000 rank6 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1112 | Classification_binary: 0.3058 | Regression_cos_phi: 0.1281 | Regression_energy: 0.1328 | Regression_eta: 0.0923 | Regression_pt: 0.1371 | Regression_sin_phi: 0.1260 | LR: 9.48e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7833s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank6 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank6 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0984 | Classification_binary: 0.2481 | Regression_cos_phi: 0.1594 | Regression_energy: 0.1509 | Regression_eta: 0.0975 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1095 | LR: 9.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4014s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank6 [2026-10-28 17:56:05,209] INFO : Step 7900/50000 rank6 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1041 | Classification_binary: 0.3238 | Regression_cos_phi: 0.1529 | Regression_energy: 0.1614 | Regression_eta: 0.1057 | Regression_pt: 0.1682 | Regression_sin_phi: 0.1947 | LR: 9.46e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.3974s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank6 [2026-10-28 18:00:55,278] INFO : Step 8000/50000 rank6 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1401 | Classification_binary: 0.3274 | Regression_cos_phi: 0.1775 | Regression_energy: 0.1501 | Regression_eta: 0.1146 | Regression_pt: 0.1561 | Regression_sin_phi: 0.1805 | LR: 9.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4832s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank6 [2026-10-28 18:00:56,228] INFO : Running validation on rank6 +rank6 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:01:47,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:01:48,313] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 18:02:14,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:02:15,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:02:16,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:02:17,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:02:27,914] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:02:28,859] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 18:02:44,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:02:45,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:02:46,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:02:47,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:02:57,878] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:02:58,281] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:02:58,745] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 18:03:14,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:03:15,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:03:16,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:03:16,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:03:26,926] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 18:08:30,022] INFO : Step 8100/50000 rank6 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2468 | Regression_cos_phi: 0.1210 | Regression_energy: 0.1383 | Regression_eta: 0.1057 | Regression_pt: 0.1406 | Regression_sin_phi: 0.1158 | LR: 9.43e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3428s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank6 [2026-10-28 18:13:41,922] INFO : Step 8200/50000 rank6 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1161 | Classification_binary: 0.2864 | Regression_cos_phi: 0.1614 | Regression_energy: 0.1399 | Regression_eta: 0.0970 | Regression_pt: 0.1426 | Regression_sin_phi: 0.1481 | LR: 9.42e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3266s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank6 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank6 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.3169 | Regression_cos_phi: 0.1300 | Regression_energy: 0.1361 | Regression_eta: 0.1104 | Regression_pt: 0.1423 | Regression_sin_phi: 0.1286 | LR: 9.40e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6359s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank6 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank6 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2533 | Regression_cos_phi: 0.1301 | Regression_energy: 0.1325 | Regression_eta: 0.0929 | Regression_pt: 0.1370 | Regression_sin_phi: 0.1326 | LR: 9.39e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.1826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank6 [2026-10-28 18:28:52,132] INFO : Step 8500/50000 rank6 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1026 | Classification_binary: 0.2328 | Regression_cos_phi: 0.1224 | Regression_energy: 0.1182 | Regression_eta: 0.0967 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1454 | LR: 9.37e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5046s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank6 [2026-10-28 18:34:09,472] INFO : Step 8600/50000 rank6 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0928 | Classification_binary: 0.2964 | Regression_cos_phi: 0.1124 | Regression_energy: 0.1121 | Regression_eta: 0.0940 | Regression_pt: 0.1178 | Regression_sin_phi: 0.1414 | LR: 9.36e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7742s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank6 [2026-10-28 18:39:14,579] INFO : Step 8700/50000 rank6 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2647 | Regression_cos_phi: 0.1495 | Regression_energy: 0.1245 | Regression_eta: 0.1057 | Regression_pt: 0.1259 | Regression_sin_phi: 0.1269 | LR: 9.34e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.7188s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank6 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank6 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2536 | Regression_cos_phi: 0.1366 | Regression_energy: 0.1257 | Regression_eta: 0.0948 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0897 | LR: 9.33e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.4099s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank6 [2026-10-28 18:49:28,517] INFO : Step 8900/50000 rank6 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 0.3510 | Regression_cos_phi: 0.1577 | Regression_energy: 0.1304 | Regression_eta: 0.1071 | Regression_pt: 0.1365 | Regression_sin_phi: 0.1539 | LR: 9.31e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7922s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank6 [2026-10-28 18:54:53,970] INFO : Step 9000/50000 rank6 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1033 | Classification_binary: 0.2741 | Regression_cos_phi: 0.1380 | Regression_energy: 0.1265 | Regression_eta: 0.1027 | Regression_pt: 0.1318 | Regression_sin_phi: 0.1127 | LR: 9.30e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.8386s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank6 [2026-10-28 18:54:54,936] INFO : Running validation on rank6 +rank6 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:55:52,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:55:53,257] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 18:56:19,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:56:21,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:56:23,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:56:24,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:56:34,142] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:56:35,474] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 18:56:51,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:56:52,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:56:53,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:56:54,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:57:04,465] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:57:05,414] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:57:05,960] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 18:57:21,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:57:22,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:57:23,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:57:24,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:57:34,106] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 19:03:08,598] INFO : Step 9100/50000 rank6 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2133 | Regression_cos_phi: 0.1261 | Regression_energy: 0.1059 | Regression_eta: 0.0891 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1098 | LR: 9.28e-04 | DataLoad Time: 0.0180s | Model Forward Time: 6.1689s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank6 [2026-10-28 19:08:18,222] INFO : Step 9200/50000 rank6 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1319 | Classification_binary: 0.2920 | Regression_cos_phi: 0.1303 | Regression_energy: 0.1387 | Regression_eta: 0.1199 | Regression_pt: 0.1455 | Regression_sin_phi: 0.1691 | LR: 9.27e-04 | DataLoad Time: 0.0234s | Model Forward Time: 6.5454s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank6 [2026-10-28 19:13:30,254] INFO : Step 9300/50000 rank6 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1279 | Classification_binary: 0.3521 | Regression_cos_phi: 0.1739 | Regression_energy: 0.1322 | Regression_eta: 0.1177 | Regression_pt: 0.1391 | Regression_sin_phi: 0.1528 | LR: 9.25e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3919s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank6 [2026-10-28 19:18:55,614] INFO : Step 9400/50000 rank6 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2516 | Regression_cos_phi: 0.1488 | Regression_energy: 0.1153 | Regression_eta: 0.0948 | Regression_pt: 0.1194 | Regression_sin_phi: 0.1294 | LR: 9.24e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.2012s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank6 [2026-10-28 19:24:20,792] INFO : Step 9500/50000 rank6 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0942 | Classification_binary: 0.2552 | Regression_cos_phi: 0.1372 | Regression_energy: 0.1252 | Regression_eta: 0.1249 | Regression_pt: 0.1261 | Regression_sin_phi: 0.2120 | LR: 9.22e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2857s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank6 [2026-10-28 19:29:34,439] INFO : Step 9600/50000 rank6 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2811 | Regression_cos_phi: 0.1256 | Regression_energy: 0.1242 | Regression_eta: 0.0980 | Regression_pt: 0.1321 | Regression_sin_phi: 0.1605 | LR: 9.21e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6054s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank6 [2026-10-28 19:34:53,791] INFO : Step 9700/50000 rank6 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2860 | Regression_cos_phi: 0.0854 | Regression_energy: 0.1245 | Regression_eta: 0.0862 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0979 | LR: 9.19e-04 | DataLoad Time: 0.0124s | Model Forward Time: 4.9680s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank6 [2026-10-28 19:39:53,937] INFO : Step 9800/50000 rank6 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0922 | Classification_binary: 0.2582 | Regression_cos_phi: 0.1218 | Regression_energy: 0.1258 | Regression_eta: 0.0866 | Regression_pt: 0.1304 | Regression_sin_phi: 0.1297 | LR: 9.17e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5615s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank6 [2026-10-28 19:44:46,548] INFO : Step 9900/50000 rank6 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2513 | Regression_cos_phi: 0.1306 | Regression_energy: 0.1229 | Regression_eta: 0.0789 | Regression_pt: 0.1284 | Regression_sin_phi: 0.1020 | LR: 9.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9320s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank6 [2026-10-28 19:49:37,631] INFO : Step 10000/50000 rank6 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2895 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1177 | Regression_eta: 0.1049 | Regression_pt: 0.1231 | Regression_sin_phi: 0.1519 | LR: 9.14e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.6247s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank6 [2026-10-28 19:49:38,691] INFO : Running validation on rank6 +rank6 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 19:50:35,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:50:35,625] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 19:51:01,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 19:51:02,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 19:51:03,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 19:51:05,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 19:51:15,147] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 19:51:15,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:51:16,369] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 19:51:32,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 19:51:33,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 19:51:34,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 19:51:35,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 19:51:45,533] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 19:51:46,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:51:46,686] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 19:52:02,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 19:52:03,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 19:52:03,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 19:52:04,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 19:52:14,763] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 19:57:27,968] INFO : Step 10100/50000 rank6 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2292 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1031 | Regression_eta: 0.0752 | Regression_pt: 0.1095 | Regression_sin_phi: 0.1374 | LR: 9.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7461s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank6 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank6 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2315 | Regression_cos_phi: 0.1617 | Regression_energy: 0.0954 | Regression_eta: 0.1259 | Regression_pt: 0.1000 | Regression_sin_phi: 0.1429 | LR: 9.11e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6807s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank6 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank6 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2797 | Regression_cos_phi: 0.1148 | Regression_energy: 0.1120 | Regression_eta: 0.0777 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0914 | LR: 9.09e-04 | DataLoad Time: 0.0232s | Model Forward Time: 5.1985s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank6 [2026-10-28 20:12:49,149] INFO : Step 10400/50000 rank6 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2547 | Regression_cos_phi: 0.1431 | Regression_energy: 0.0998 | Regression_eta: 0.0878 | Regression_pt: 0.1065 | Regression_sin_phi: 0.1132 | LR: 9.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2227s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank6 [2026-10-28 20:18:09,149] INFO : Step 10500/50000 rank6 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0909 | Regression_energy: 0.1264 | Regression_eta: 0.0606 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0775 | LR: 9.06e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3228s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank6 [2026-10-28 20:23:17,055] INFO : Step 10600/50000 rank6 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2709 | Regression_cos_phi: 0.1345 | Regression_energy: 0.1226 | Regression_eta: 0.0708 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0979 | LR: 9.04e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0923s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank6 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank6 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2692 | Regression_cos_phi: 0.1029 | Regression_energy: 0.1289 | Regression_eta: 0.0830 | Regression_pt: 0.1315 | Regression_sin_phi: 0.1357 | LR: 9.02e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8713s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank6 [2026-10-28 20:33:28,995] INFO : Step 10800/50000 rank6 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0990 | Regression_energy: 0.1219 | Regression_eta: 0.0861 | Regression_pt: 0.1254 | Regression_sin_phi: 0.1515 | LR: 9.00e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.6821s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank6 [2026-10-28 20:38:44,062] INFO : Step 10900/50000 rank6 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2199 | Regression_cos_phi: 0.0869 | Regression_energy: 0.1033 | Regression_eta: 0.0633 | Regression_pt: 0.1099 | Regression_sin_phi: 0.1178 | LR: 8.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7072s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank6 [2026-10-28 20:43:46,526] INFO : Step 11000/50000 rank6 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2666 | Regression_cos_phi: 0.0942 | Regression_energy: 0.1134 | Regression_eta: 0.0771 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1147 | LR: 8.97e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4439s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank6 [2026-10-28 20:43:47,416] INFO : Running validation on rank6 +rank6 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 20:44:41,297] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 20:45:09,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 20:45:10,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 20:45:11,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 20:45:13,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 20:45:23,046] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:45:23,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:45:24,127] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 20:45:40,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 20:45:41,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 20:45:42,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 20:45:44,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 20:45:54,014] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 20:46:11,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 20:46:12,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 20:46:13,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 20:46:14,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 20:46:24,686] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:51:55,949] INFO : Step 11100/50000 rank6 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.2406 | Regression_cos_phi: 0.0660 | Regression_energy: 0.1073 | Regression_eta: 0.0615 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0929 | LR: 8.95e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.8312s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank6 [2026-10-28 20:57:10,091] INFO : Step 11200/50000 rank6 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2094 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1124 | Regression_eta: 0.0692 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0955 | LR: 8.93e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.8527s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank6 [2026-10-28 21:02:18,461] INFO : Step 11300/50000 rank6 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2518 | Regression_cos_phi: 0.1232 | Regression_energy: 0.1016 | Regression_eta: 0.0764 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1102 | LR: 8.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank6 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank6 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2314 | Regression_cos_phi: 0.1056 | Regression_energy: 0.1057 | Regression_eta: 0.0818 | Regression_pt: 0.1088 | Regression_sin_phi: 0.1350 | LR: 8.89e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.9835s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank6 [2026-10-28 21:12:15,938] INFO : Step 11500/50000 rank6 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0933 | Regression_energy: 0.1164 | Regression_eta: 0.0540 | Regression_pt: 0.1198 | Regression_sin_phi: 0.1046 | LR: 8.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.3384s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank6 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank6 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2553 | Regression_cos_phi: 0.1314 | Regression_energy: 0.0971 | Regression_eta: 0.0686 | Regression_pt: 0.1043 | Regression_sin_phi: 0.1050 | LR: 8.86e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2580s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank6 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank6 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2573 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1004 | Regression_eta: 0.0784 | Regression_pt: 0.1024 | Regression_sin_phi: 0.1120 | LR: 8.84e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5947s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank6 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank6 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0550 | Regression_energy: 0.1088 | Regression_eta: 0.0796 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0781 | LR: 8.82e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5587s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank6 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank6 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2319 | Regression_cos_phi: 0.1422 | Regression_energy: 0.1037 | Regression_eta: 0.0735 | Regression_pt: 0.1067 | Regression_sin_phi: 0.1280 | LR: 8.80e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9859s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank6 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank6 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.2678 | Regression_cos_phi: 0.1038 | Regression_energy: 0.1233 | Regression_eta: 0.0605 | Regression_pt: 0.1304 | Regression_sin_phi: 0.1034 | LR: 8.78e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0213s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank6 [2026-10-28 21:36:38,362] INFO : Running validation on rank6 +rank6 [2026-10-28 21:36:38,362] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:37:31,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:37:31,909] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 21:38:00,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:38:01,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:38:02,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:38:03,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:38:13,526] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:38:14,611] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 21:38:30,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:38:32,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:38:34,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:38:35,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:38:45,015] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 21:38:45,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:38:46,116] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 21:39:01,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:39:02,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:39:03,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:39:04,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:39:14,256] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank6 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.1998 | Regression_cos_phi: 0.1128 | Regression_energy: 0.0960 | Regression_eta: 0.0560 | Regression_pt: 0.0984 | Regression_sin_phi: 0.1078 | LR: 8.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1504s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank6 [2026-10-28 21:49:00,967] INFO : Step 12200/50000 rank6 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0869 | Regression_eta: 0.0505 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0879 | LR: 8.74e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5141s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank6 [2026-10-28 21:53:55,880] INFO : Step 12300/50000 rank6 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2786 | Regression_cos_phi: 0.1257 | Regression_energy: 0.1020 | Regression_eta: 0.0901 | Regression_pt: 0.1056 | Regression_sin_phi: 0.1252 | LR: 8.72e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6878s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank6 [2026-10-28 21:58:40,948] INFO : Step 12400/50000 rank6 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2435 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0987 | Regression_eta: 0.0743 | Regression_pt: 0.1016 | Regression_sin_phi: 0.1215 | LR: 8.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2700s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank6 [2026-10-28 22:03:23,386] INFO : Step 12500/50000 rank6 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0788 | Regression_eta: 0.0628 | Regression_pt: 0.0795 | Regression_sin_phi: 0.1228 | LR: 8.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6721s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank6 [2026-10-28 22:08:04,812] INFO : Step 12600/50000 rank6 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2614 | Regression_cos_phi: 0.1157 | Regression_energy: 0.0954 | Regression_eta: 0.0735 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1056 | LR: 8.66e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.1047s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank6 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank6 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.2736 | Regression_cos_phi: 0.1303 | Regression_energy: 0.0949 | Regression_eta: 0.0832 | Regression_pt: 0.0973 | Regression_sin_phi: 0.1163 | LR: 8.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3601s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank6 [2026-10-28 22:17:38,512] INFO : Step 12800/50000 rank6 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2321 | Regression_cos_phi: 0.1214 | Regression_energy: 0.0949 | Regression_eta: 0.0768 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0964 | LR: 8.62e-04 | DataLoad Time: 0.0243s | Model Forward Time: 6.0949s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank6 [2026-10-28 22:22:32,703] INFO : Step 12900/50000 rank6 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2525 | Regression_cos_phi: 0.1472 | Regression_energy: 0.1012 | Regression_eta: 0.1035 | Regression_pt: 0.1031 | Regression_sin_phi: 0.1439 | LR: 8.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6630s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank6 [2026-10-28 22:27:21,819] INFO : Step 13000/50000 rank6 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.2862 | Regression_cos_phi: 0.1206 | Regression_energy: 0.0942 | Regression_eta: 0.1201 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1290 | LR: 8.58e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.9257s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank6 [2026-10-28 22:27:23,211] INFO : Running validation on rank6 +rank6 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:28:28,169] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 22:28:54,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 22:28:55,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 22:28:56,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 22:28:57,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 22:29:07,391] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:29:08,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:29:08,605] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 22:29:24,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 22:29:25,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 22:29:26,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 22:29:27,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 22:29:37,498] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:29:38,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:29:38,493] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 22:29:54,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 22:29:55,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 22:29:56,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 22:29:56,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 22:30:06,936] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 22:35:15,707] INFO : Step 13100/50000 rank6 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0675 | Regression_eta: 0.0597 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0742 | LR: 8.56e-04 | DataLoad Time: 0.0226s | Model Forward Time: 6.4832s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank6 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank6 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1935 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0845 | Regression_eta: 0.0498 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0743 | LR: 8.54e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.6856s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank6 [2026-10-28 22:44:58,539] INFO : Step 13300/50000 rank6 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.2497 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0852 | Regression_eta: 0.0659 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0770 | LR: 8.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6742s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank6 [2026-10-28 22:49:50,191] INFO : Step 13400/50000 rank6 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2926 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1235 | Regression_eta: 0.1050 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1142 | LR: 8.50e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.6023s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank6 [2026-10-28 22:54:39,510] INFO : Step 13500/50000 rank6 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2155 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0919 | Regression_eta: 0.0473 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0686 | LR: 8.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4632s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank6 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank6 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2069 | Regression_cos_phi: 0.1183 | Regression_energy: 0.0819 | Regression_eta: 0.0668 | Regression_pt: 0.0825 | Regression_sin_phi: 0.1086 | LR: 8.45e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8498s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank6 [2026-10-28 23:04:05,988] INFO : Step 13700/50000 rank6 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2902 | Regression_cos_phi: 0.1013 | Regression_energy: 0.0972 | Regression_eta: 0.0854 | Regression_pt: 0.1028 | Regression_sin_phi: 0.1465 | LR: 8.43e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7312s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank6 [2026-10-28 23:09:16,294] INFO : Step 13800/50000 rank6 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2083 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0990 | Regression_eta: 0.0642 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0780 | LR: 8.41e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4790s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank6 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank6 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1201 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0594 | Regression_energy: 0.1222 | Regression_eta: 0.0469 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0844 | LR: 8.39e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank6 [2026-10-28 23:18:59,721] INFO : Step 14000/50000 rank6 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2612 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0929 | Regression_eta: 0.0565 | Regression_pt: 0.0970 | Regression_sin_phi: 0.1072 | LR: 8.37e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.1777s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank6 [2026-10-28 23:19:00,860] INFO : Running validation on rank6 +rank6 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:19:00,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 23:19:52,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:19:53,516] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-28 23:20:19,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 23:20:20,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 23:20:21,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 23:20:22,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 23:20:32,627] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:20:33,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:20:33,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-28 23:20:49,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 23:20:50,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 23:20:51,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 23:20:52,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 23:21:02,302] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:21:03,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:21:04,037] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-28 23:21:19,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 23:21:20,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 23:21:21,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 23:21:22,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 23:21:32,306] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 23:26:43,450] INFO : Step 14100/50000 rank6 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0852 | Regression_eta: 0.0869 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0624 | LR: 8.35e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5268s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank6 [2026-10-28 23:31:31,720] INFO : Step 14200/50000 rank6 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0905 | Regression_eta: 0.0542 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0774 | LR: 8.32e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6179s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank6 [2026-10-28 23:36:16,487] INFO : Step 14300/50000 rank6 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0889 | Regression_eta: 0.0665 | Regression_pt: 0.0895 | Regression_sin_phi: 0.1051 | LR: 8.30e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank6 [2026-10-28 23:40:58,337] INFO : Step 14400/50000 rank6 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2448 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0875 | Regression_eta: 0.0585 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0893 | LR: 8.28e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7758s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank6 [2026-10-28 23:45:42,941] INFO : Step 14500/50000 rank6 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0744 | Regression_eta: 0.0528 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0807 | LR: 8.26e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.6346s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank6 [2026-10-28 23:50:27,494] INFO : Step 14600/50000 rank6 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2467 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0709 | Regression_eta: 0.0554 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0992 | LR: 8.24e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4510s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank6 [2026-10-28 23:55:12,844] INFO : Step 14700/50000 rank6 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2399 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0799 | Regression_eta: 0.0581 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0839 | LR: 8.21e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6472s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank6 [2026-10-28 23:59:56,486] INFO : Step 14800/50000 rank6 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.2148 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0938 | Regression_eta: 0.0656 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0902 | LR: 8.19e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.7604s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank6 [2026-10-29 00:04:46,438] INFO : Step 14900/50000 rank6 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0790 | Regression_eta: 0.0331 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0810 | LR: 8.17e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1963s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank6 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank6 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2520 | Regression_cos_phi: 0.1468 | Regression_energy: 0.0982 | Regression_eta: 0.0879 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0823 | LR: 8.15e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7487s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank6 [2026-10-29 00:09:44,171] INFO : Running validation on rank6 +rank6 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:10:38,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:10:39,314] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 00:11:05,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:11:07,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:11:09,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:11:10,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:11:20,261] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 00:11:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:11:20,773] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 00:11:37,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:11:38,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:11:39,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:11:40,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:11:50,222] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:11:50,986] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 00:12:06,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:12:07,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:12:08,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:12:09,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:12:19,393] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 00:17:30,254] INFO : Step 15100/50000 rank6 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2374 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0957 | Regression_eta: 0.0592 | Regression_pt: 0.1022 | Regression_sin_phi: 0.1047 | LR: 8.12e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3988s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank6 [2026-10-29 00:22:16,917] INFO : Step 15200/50000 rank6 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0709 | Regression_eta: 0.0462 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0469 | LR: 8.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2248s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank6 [2026-10-29 00:27:05,718] INFO : Step 15300/50000 rank6 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0887 | Regression_eta: 0.0502 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0932 | LR: 8.08e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3160s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank6 [2026-10-29 00:31:57,727] INFO : Step 15400/50000 rank6 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2494 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0851 | Regression_eta: 0.0619 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0830 | LR: 8.05e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6651s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank6 [2026-10-29 00:36:52,790] INFO : Step 15500/50000 rank6 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0848 | Regression_energy: 0.1027 | Regression_eta: 0.0454 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0787 | LR: 8.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4222s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank6 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank6 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0919 | Regression_eta: 0.0410 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0713 | LR: 8.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9286s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank6 [2026-10-29 00:46:43,872] INFO : Step 15700/50000 rank6 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2413 | Regression_cos_phi: 0.1191 | Regression_energy: 0.0853 | Regression_eta: 0.0781 | Regression_pt: 0.0890 | Regression_sin_phi: 0.1170 | LR: 7.98e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7557s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank6 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank6 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1944 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0824 | Regression_eta: 0.0586 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0904 | LR: 7.96e-04 | DataLoad Time: 0.0242s | Model Forward Time: 5.0238s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank6 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank6 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0594 | Regression_eta: 0.0643 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0593 | LR: 7.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1825s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank6 [2026-10-29 01:01:16,825] INFO : Step 16000/50000 rank6 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2299 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0798 | Regression_eta: 0.0548 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0826 | LR: 7.91e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.3027s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank6 [2026-10-29 01:01:17,883] INFO : Running validation on rank6 +rank6 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 01:02:10,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:02:11,391] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 01:02:39,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 01:02:40,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 01:02:42,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 01:02:44,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 01:02:54,806] INFO : Time taken to make predictions on device 6 is: 0.29 min +rank6 [2026-10-29 01:02:55,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:02:55,636] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 01:03:11,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 01:03:12,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 01:03:13,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 01:03:14,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 01:03:24,858] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:03:25,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:03:26,085] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 01:03:41,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 01:03:42,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 01:03:43,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 01:03:44,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 01:03:54,378] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank6 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.2088 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0661 | Regression_eta: 0.0486 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0668 | LR: 7.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0649s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank6 [2026-10-29 01:14:08,172] INFO : Step 16200/50000 rank6 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0796 | Regression_eta: 0.0433 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0681 | LR: 7.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2122s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank6 [2026-10-29 01:19:03,891] INFO : Step 16300/50000 rank6 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2352 | Regression_cos_phi: 0.1050 | Regression_energy: 0.1073 | Regression_eta: 0.0787 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0988 | LR: 7.84e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.2098s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank6 [2026-10-29 01:23:56,183] INFO : Step 16400/50000 rank6 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0932 | Regression_eta: 0.0535 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0793 | LR: 7.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6831s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank6 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank6 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0747 | Regression_eta: 0.0541 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0810 | LR: 7.79e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3079s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank6 [2026-10-29 01:33:46,142] INFO : Step 16600/50000 rank6 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0931 | Regression_eta: 0.0526 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0719 | LR: 7.77e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1712s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank6 [2026-10-29 01:38:38,620] INFO : Step 16700/50000 rank6 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0927 | Regression_eta: 0.0721 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0826 | LR: 7.74e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.9119s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank6 [2026-10-29 01:43:30,909] INFO : Step 16800/50000 rank6 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0688 | Regression_eta: 0.0549 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0971 | LR: 7.72e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.5997s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank6 [2026-10-29 01:48:23,689] INFO : Step 16900/50000 rank6 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0837 | Regression_eta: 0.0309 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0712 | LR: 7.69e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3815s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank6 [2026-10-29 01:53:19,526] INFO : Step 17000/50000 rank6 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2394 | Regression_cos_phi: 0.1090 | Regression_energy: 0.0734 | Regression_eta: 0.0823 | Regression_pt: 0.0762 | Regression_sin_phi: 0.1266 | LR: 7.67e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.6728s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank6 [2026-10-29 01:53:20,625] INFO : Running validation on rank6 +rank6 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 01:54:26,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:54:27,391] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 01:55:04,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 01:55:05,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 01:55:06,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 01:55:07,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 01:55:17,860] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:55:18,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 01:55:34,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 01:55:35,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 01:55:36,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 01:55:38,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 01:55:48,162] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:55:50,415] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 01:56:06,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 01:56:07,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 01:56:07,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 01:56:08,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 01:56:18,817] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 02:01:32,244] INFO : Step 17100/50000 rank6 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0746 | Regression_eta: 0.0399 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0722 | LR: 7.64e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.1170s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank6 [2026-10-29 02:06:24,508] INFO : Step 17200/50000 rank6 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0756 | Regression_eta: 0.0557 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0868 | LR: 7.62e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.6295s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank6 [2026-10-29 02:11:06,510] INFO : Step 17300/50000 rank6 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1979 | Regression_cos_phi: 0.1277 | Regression_energy: 0.0667 | Regression_eta: 0.0485 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0835 | LR: 7.59e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7542s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank6 [2026-10-29 02:16:07,514] INFO : Step 17400/50000 rank6 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2365 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0750 | Regression_eta: 0.0480 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0665 | LR: 7.57e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.5477s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank6 [2026-10-29 02:21:02,097] INFO : Step 17500/50000 rank6 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0771 | Regression_eta: 0.0499 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0851 | LR: 7.54e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2410s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank6 [2026-10-29 02:25:59,309] INFO : Step 17600/50000 rank6 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0671 | Regression_eta: 0.0610 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0807 | LR: 7.52e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4714s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank6 [2026-10-29 02:30:47,052] INFO : Step 17700/50000 rank6 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0787 | Regression_eta: 0.0520 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0695 | LR: 7.49e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7713s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank6 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank6 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0640 | Regression_eta: 0.0373 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0614 | LR: 7.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0200s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank6 [2026-10-29 02:40:30,747] INFO : Step 17900/50000 rank6 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0751 | Regression_eta: 0.0454 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0461 | LR: 7.44e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5940s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank6 [2026-10-29 02:45:16,091] INFO : Step 18000/50000 rank6 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2400 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0739 | Regression_eta: 0.0534 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0772 | LR: 7.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6584s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank6 [2026-10-29 02:45:17,190] INFO : Running validation on rank6 +rank6 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:46:12,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:46:13,322] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 02:46:39,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:46:40,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:46:41,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:46:42,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:46:52,689] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:46:53,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 02:47:09,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:47:10,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:47:12,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:47:13,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:47:23,769] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 02:47:25,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:47:25,821] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 02:47:42,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:47:42,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:47:43,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:47:44,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:47:54,594] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank6 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.2136 | Regression_cos_phi: 0.1054 | Regression_energy: 0.0634 | Regression_eta: 0.0683 | Regression_pt: 0.0654 | Regression_sin_phi: 0.1048 | LR: 7.39e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7394s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank6 [2026-10-29 02:58:08,645] INFO : Step 18200/50000 rank6 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0808 | Regression_eta: 0.0546 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0592 | LR: 7.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9665s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank6 [2026-10-29 03:02:55,066] INFO : Step 18300/50000 rank6 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1913 | Regression_cos_phi: 0.1013 | Regression_energy: 0.0797 | Regression_eta: 0.0660 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0968 | LR: 7.34e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5938s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank6 [2026-10-29 03:07:45,978] INFO : Step 18400/50000 rank6 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2374 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0778 | Regression_eta: 0.0553 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0725 | LR: 7.31e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.7147s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank6 [2026-10-29 03:12:43,565] INFO : Step 18500/50000 rank6 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0689 | Regression_eta: 0.0428 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0647 | LR: 7.29e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.3590s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank6 [2026-10-29 03:17:39,475] INFO : Step 18600/50000 rank6 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0785 | Regression_eta: 0.0583 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0768 | LR: 7.26e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.4732s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank6 [2026-10-29 03:22:31,157] INFO : Step 18700/50000 rank6 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0643 | Regression_eta: 0.0559 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0541 | LR: 7.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4803s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank6 [2026-10-29 03:27:37,244] INFO : Step 18800/50000 rank6 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0771 | Regression_eta: 0.0674 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0779 | LR: 7.21e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.9519s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank6 [2026-10-29 03:32:26,943] INFO : Step 18900/50000 rank6 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0853 | Regression_eta: 0.0416 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0710 | LR: 7.18e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5367s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank6 [2026-10-29 03:37:06,569] INFO : Step 19000/50000 rank6 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0984 | Regression_energy: 0.0666 | Regression_eta: 0.0592 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0846 | LR: 7.16e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5303s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank6 [2026-10-29 03:37:07,713] INFO : Running validation on rank6 +rank6 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 03:38:00,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:38:00,820] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 03:38:27,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 03:38:28,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 03:38:29,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 03:38:30,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 03:38:40,567] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 03:38:40,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:38:41,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 03:38:57,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 03:38:58,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 03:38:59,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 03:39:00,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 03:39:10,396] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:39:11,231] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 03:39:26,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 03:39:27,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 03:39:28,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 03:39:29,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 03:39:39,435] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 03:44:42,723] INFO : Step 19100/50000 rank6 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.2031 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0679 | Regression_eta: 0.0453 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0770 | LR: 7.13e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9852s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank6 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank6 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0604 | Regression_eta: 0.0330 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0676 | LR: 7.10e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5593s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank6 [2026-10-29 03:54:09,159] INFO : Step 19300/50000 rank6 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0778 | Regression_eta: 0.0573 | Regression_pt: 0.0798 | Regression_sin_phi: 0.1323 | LR: 7.08e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3958s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank6 [2026-10-29 03:58:51,217] INFO : Step 19400/50000 rank6 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2044 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0845 | Regression_eta: 0.0471 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0785 | LR: 7.05e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6174s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank6 [2026-10-29 04:03:30,643] INFO : Step 19500/50000 rank6 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0663 | Regression_eta: 0.0578 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0898 | LR: 7.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank6 [2026-10-29 04:08:12,466] INFO : Step 19600/50000 rank6 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0681 | Regression_eta: 0.0474 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0531 | LR: 7.00e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.4180s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank6 [2026-10-29 04:12:52,349] INFO : Step 19700/50000 rank6 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0847 | Regression_eta: 0.0484 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0855 | LR: 6.97e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.3598s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank6 [2026-10-29 04:17:28,315] INFO : Step 19800/50000 rank6 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0685 | Regression_eta: 0.0543 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0962 | LR: 6.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0463s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank6 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank6 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0834 | Regression_energy: 0.1030 | Regression_eta: 0.0428 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0819 | LR: 6.92e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.3166s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank6 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank6 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0761 | Regression_eta: 0.0494 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0788 | LR: 6.89e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6555s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank6 [2026-10-29 04:26:52,970] INFO : Running validation on rank6 +rank6 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:27:46,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:27:46,652] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 04:28:14,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:28:15,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:28:16,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:28:17,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:28:27,316] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:28:28,502] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 04:28:44,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:28:45,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:28:47,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:28:48,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:28:58,145] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 04:28:58,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:28:59,252] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 04:29:14,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:29:15,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:29:16,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:29:17,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:29:27,352] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 04:34:27,055] INFO : Step 20100/50000 rank6 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0660 | Regression_eta: 0.0510 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0919 | LR: 6.86e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.3244s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank6 [2026-10-29 04:39:08,146] INFO : Step 20200/50000 rank6 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0622 | Regression_eta: 0.0507 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0913 | LR: 6.84e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6157s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank6 [2026-10-29 04:43:47,102] INFO : Step 20300/50000 rank6 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1171 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0446 | Regression_eta: 0.0250 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0716 | LR: 6.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4957s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank6 [2026-10-29 04:48:26,732] INFO : Step 20400/50000 rank6 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0655 | Regression_eta: 0.0728 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0764 | LR: 6.78e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4418s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank6 [2026-10-29 04:53:08,466] INFO : Step 20500/50000 rank6 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2129 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0696 | Regression_eta: 0.0524 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0713 | LR: 6.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0616s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank6 [2026-10-29 04:57:53,290] INFO : Step 20600/50000 rank6 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0688 | Regression_eta: 0.0519 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0505 | LR: 6.73e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7529s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank6 [2026-10-29 05:02:47,889] INFO : Step 20700/50000 rank6 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0689 | Regression_eta: 0.0384 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0521 | LR: 6.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6375s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank6 [2026-10-29 05:07:34,620] INFO : Step 20800/50000 rank6 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2242 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0567 | Regression_eta: 0.0564 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0641 | LR: 6.67e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.8143s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank6 [2026-10-29 05:12:18,487] INFO : Step 20900/50000 rank6 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0725 | Regression_eta: 0.0404 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0414 | LR: 6.65e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.6281s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank6 [2026-10-29 05:17:03,998] INFO : Step 21000/50000 rank6 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0620 | Regression_eta: 0.0405 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0740 | LR: 6.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank6 [2026-10-29 05:17:04,884] INFO : Running validation on rank6 +rank6 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 05:17:58,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:17:58,801] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 05:18:22,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 05:18:23,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 05:18:25,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 05:18:26,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 05:18:36,074] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:18:37,118] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 05:18:52,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 05:18:53,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 05:18:54,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 05:18:55,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 05:19:05,774] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:19:06,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:19:07,254] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 05:19:22,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 05:19:23,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 05:19:24,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 05:19:25,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 05:19:35,462] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank6 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0662 | Regression_eta: 0.0533 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0930 | LR: 6.59e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.3803s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank6 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank6 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 05:29:24,522] INFO : Step 21200/50000 rank6 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0813 | Regression_eta: 0.0407 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0578 | LR: 6.56e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6427s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank6 [2026-10-29 05:34:39,116] INFO : Step 21300/50000 rank6 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0678 | Regression_eta: 0.0501 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0641 | LR: 6.54e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6729s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank6 [2026-10-29 05:39:29,690] INFO : Step 21400/50000 rank6 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0650 | Regression_eta: 0.0491 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0574 | LR: 6.51e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.3963s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank6 [2026-10-29 05:44:17,735] INFO : Step 21500/50000 rank6 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0533 | Regression_eta: 0.0630 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0678 | LR: 6.48e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3076s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank6 [2026-10-29 05:49:11,031] INFO : Step 21600/50000 rank6 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0689 | Regression_eta: 0.0395 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0749 | LR: 6.45e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5071s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank6 [2026-10-29 05:53:59,566] INFO : Step 21700/50000 rank6 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2164 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0638 | Regression_eta: 0.0602 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0898 | LR: 6.43e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5561s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank6 [2026-10-29 05:58:52,460] INFO : Step 21800/50000 rank6 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0783 | Regression_energy: 0.0543 | Regression_eta: 0.0448 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0634 | LR: 6.40e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1505s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank6 [2026-10-29 06:03:49,408] INFO : Step 21900/50000 rank6 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0730 | Regression_eta: 0.0484 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0595 | LR: 6.37e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8519s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank6 [2026-10-29 06:08:50,709] INFO : Step 22000/50000 rank6 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0638 | Regression_eta: 0.0475 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0753 | LR: 6.34e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.4744s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank6 [2026-10-29 06:08:51,806] INFO : Running validation on rank6 +rank6 [2026-10-29 06:08:51,806] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 06:08:51,806] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 06:09:45,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:09:46,125] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 06:10:12,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 06:10:13,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 06:10:14,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 06:10:16,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 06:10:26,181] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 06:10:26,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 06:10:43,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 06:10:44,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 06:10:45,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 06:10:46,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 06:10:56,695] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:10:56,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:10:57,441] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 06:11:13,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 06:11:13,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 06:11:14,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 06:11:15,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 06:11:25,733] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 06:16:42,857] INFO : Step 22100/50000 rank6 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0696 | Regression_eta: 0.0615 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0722 | LR: 6.32e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1887s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank6 [2026-10-29 06:21:31,723] INFO : Step 22200/50000 rank6 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0684 | Regression_eta: 0.0337 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0649 | LR: 6.29e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.2921s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank6 [2026-10-29 06:26:33,413] INFO : Step 22300/50000 rank6 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0603 | Regression_eta: 0.0314 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0358 | LR: 6.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7527s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank6 [2026-10-29 06:31:29,808] INFO : Step 22400/50000 rank6 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0737 | Regression_eta: 0.0386 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0633 | LR: 6.23e-04 | DataLoad Time: 0.0116s | Model Forward Time: 6.3941s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank6 [2026-10-29 06:36:31,515] INFO : Step 22500/50000 rank6 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0725 | Regression_eta: 0.0625 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0701 | LR: 6.20e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2550s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank6 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank6 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0636 | Regression_eta: 0.0358 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0601 | LR: 6.18e-04 | DataLoad Time: 0.0243s | Model Forward Time: 4.3131s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank6 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank6 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0597 | Regression_eta: 0.0335 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0625 | LR: 6.15e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9428s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank6 [2026-10-29 06:51:26,826] INFO : Step 22800/50000 rank6 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0638 | Regression_eta: 0.0477 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0739 | LR: 6.12e-04 | DataLoad Time: 0.0226s | Model Forward Time: 5.4043s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank6 [2026-10-29 06:56:23,354] INFO : Step 22900/50000 rank6 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0618 | Regression_eta: 0.0410 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0751 | LR: 6.09e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8102s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank6 [2026-10-29 07:01:33,057] INFO : Step 23000/50000 rank6 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0621 | Regression_eta: 0.0570 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0760 | LR: 6.06e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.4754s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank6 [2026-10-29 07:01:34,104] INFO : Running validation on rank6 +rank6 [2026-10-29 07:01:34,104] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:01:34,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:02:28,433] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 07:02:49,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:02:50,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:02:51,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:02:52,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:03:02,611] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:03:03,444] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:03:03,719] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 07:03:20,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:03:21,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:03:22,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:03:23,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:03:33,420] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 07:03:33,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:03:34,177] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 07:03:50,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:03:50,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:03:51,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:03:52,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:04:02,615] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 07:09:13,970] INFO : Step 23100/50000 rank6 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0487 | Regression_eta: 0.0413 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0653 | LR: 6.04e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1785s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank6 [2026-10-29 07:14:12,718] INFO : Step 23200/50000 rank6 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1674 | Regression_cos_phi: 0.1274 | Regression_energy: 0.0741 | Regression_eta: 0.0390 | Regression_pt: 0.0778 | Regression_sin_phi: 0.1069 | LR: 6.01e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.6103s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank6 [2026-10-29 07:19:05,798] INFO : Step 23300/50000 rank6 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.2188 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0545 | Regression_eta: 0.0565 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0756 | LR: 5.98e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8584s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank6 [2026-10-29 07:24:01,132] INFO : Step 23400/50000 rank6 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0711 | Regression_eta: 0.0574 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0808 | LR: 5.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0099s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank6 [2026-10-29 07:29:14,711] INFO : Step 23500/50000 rank6 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0470 | Regression_eta: 0.0229 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0576 | LR: 5.92e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8335s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank6 [2026-10-29 07:34:22,318] INFO : Step 23600/50000 rank6 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0653 | Regression_eta: 0.0592 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0882 | LR: 5.90e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3900s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank6 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank6 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0660 | Regression_eta: 0.0464 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0678 | LR: 5.87e-04 | DataLoad Time: 0.0134s | Model Forward Time: 5.8119s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank6 [2026-10-29 07:44:46,685] INFO : Step 23800/50000 rank6 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0712 | Regression_eta: 0.0468 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0842 | LR: 5.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8781s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank6 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank6 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0620 | Regression_eta: 0.0370 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0587 | LR: 5.81e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8172s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank6 [2026-10-29 07:54:58,898] INFO : Step 24000/50000 rank6 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0723 | Regression_eta: 0.0399 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0738 | LR: 5.78e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8547s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank6 [2026-10-29 07:54:59,832] INFO : Running validation on rank6 +rank6 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:54:59,833] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:55:54,112] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 07:56:20,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:56:21,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:56:23,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:56:24,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:56:34,074] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:56:35,052] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 07:56:51,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:56:52,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:56:53,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:56:54,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:57:04,079] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:57:05,694] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 07:57:21,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:57:22,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:57:23,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:57:24,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:57:34,122] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 08:03:04,662] INFO : Step 24100/50000 rank6 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0667 | Regression_eta: 0.0407 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0935 | LR: 5.75e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.5924s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank6 [2026-10-29 08:07:59,793] INFO : Step 24200/50000 rank6 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0717 | Regression_eta: 0.0393 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0658 | LR: 5.73e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0927s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank6 [2026-10-29 08:13:18,506] INFO : Step 24300/50000 rank6 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0721 | Regression_eta: 0.0371 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0606 | LR: 5.70e-04 | DataLoad Time: 0.0192s | Model Forward Time: 7.4912s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank6 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank6 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0622 | Regression_eta: 0.0608 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0720 | LR: 5.67e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1596s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank6 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank6 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0662 | Regression_eta: 0.0397 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0767 | LR: 5.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5202s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank6 [2026-10-29 08:28:18,515] INFO : Step 24600/50000 rank6 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0732 | Regression_eta: 0.0340 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0548 | LR: 5.61e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.7044s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank6 [2026-10-29 08:33:21,874] INFO : Step 24700/50000 rank6 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0679 | Regression_eta: 0.0468 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0846 | LR: 5.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9285s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank6 [2026-10-29 08:38:19,550] INFO : Step 24800/50000 rank6 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0607 | Regression_eta: 0.0585 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0636 | LR: 5.56e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.2077s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank6 [2026-10-29 08:43:21,524] INFO : Step 24900/50000 rank6 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0275 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0522 | Regression_eta: 0.0294 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0422 | LR: 5.53e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4725s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank6 [2026-10-29 08:48:25,835] INFO : Step 25000/50000 rank6 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0614 | Regression_eta: 0.0422 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0868 | LR: 5.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9522s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank6 [2026-10-29 08:48:26,771] INFO : Running validation on rank6 +rank6 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 08:49:20,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:49:21,558] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 08:49:51,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 08:49:53,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 08:49:54,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 08:49:55,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 08:50:05,661] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:50:06,728] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 08:50:22,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 08:50:23,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 08:50:24,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 08:50:25,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 08:50:35,736] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:50:36,988] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 08:50:52,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 08:50:53,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 08:50:54,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 08:50:55,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 08:51:05,210] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 08:56:18,538] INFO : Step 25100/50000 rank6 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0536 | Regression_eta: 0.0354 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0458 | LR: 5.47e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.3800s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank6 [2026-10-29 09:01:01,591] INFO : Step 25200/50000 rank6 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0663 | Regression_eta: 0.0473 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0688 | LR: 5.44e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8761s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank6 [2026-10-29 09:05:50,665] INFO : Step 25300/50000 rank6 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0627 | Regression_eta: 0.0541 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0736 | LR: 5.42e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank6 [2026-10-29 09:10:30,564] INFO : Step 25400/50000 rank6 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0625 | Regression_eta: 0.0444 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0896 | LR: 5.39e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.1693s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank6 [2026-10-29 09:15:11,238] INFO : Step 25500/50000 rank6 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0677 | Regression_eta: 0.0403 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0790 | LR: 5.36e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4698s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank6 [2026-10-29 09:19:56,604] INFO : Step 25600/50000 rank6 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0891 | Regression_eta: 0.0350 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0494 | LR: 5.33e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.9424s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank6 [2026-10-29 09:24:33,865] INFO : Step 25700/50000 rank6 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2118 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0598 | Regression_eta: 0.0437 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0878 | LR: 5.30e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2136s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank6 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank6 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0652 | Regression_eta: 0.0472 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0468 | LR: 5.27e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1659s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank6 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank6 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0616 | Regression_eta: 0.0436 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0462 | LR: 5.25e-04 | DataLoad Time: 0.0247s | Model Forward Time: 4.7708s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank6 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank6 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1396 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0671 | Regression_eta: 0.0453 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0664 | LR: 5.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7484s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank6 [2026-10-29 09:39:04,390] INFO : Running validation on rank6 +rank6 [2026-10-29 09:39:04,391] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:39:04,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:39:55,909] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 09:40:23,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:40:24,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:40:25,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:40:26,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:40:36,521] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:40:37,505] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 09:40:54,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:40:55,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:40:56,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:40:57,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:41:07,735] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 09:41:08,936] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:44:09,566] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 09:44:25,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:44:26,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:44:27,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:44:28,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:44:38,671] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:50:29,555] INFO : Step 26100/50000 rank6 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2223 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0595 | Regression_eta: 0.0589 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0810 | LR: 5.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1788s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank6 [2026-10-29 09:55:15,273] INFO : Step 26200/50000 rank6 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0579 | Regression_eta: 0.0362 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0447 | LR: 5.16e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.8134s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank6 [2026-10-29 09:59:57,744] INFO : Step 26300/50000 rank6 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0680 | Regression_eta: 0.0634 | Regression_pt: 0.0709 | Regression_sin_phi: 0.1079 | LR: 5.13e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.9317s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank6 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank6 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0601 | Regression_eta: 0.0547 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0606 | LR: 5.10e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.1500s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank6 [2026-10-29 10:09:23,739] INFO : Step 26500/50000 rank6 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0557 | Regression_eta: 0.0673 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0751 | LR: 5.08e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.0936s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank6 [2026-10-29 10:14:05,782] INFO : Step 26600/50000 rank6 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0703 | Regression_eta: 0.0499 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0730 | LR: 5.05e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6632s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank6 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank6 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0569 | Regression_eta: 0.0387 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0573 | LR: 5.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2141s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank6 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank6 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0560 | Regression_eta: 0.0330 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0600 | LR: 4.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5303s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank6 [2026-10-29 10:28:51,387] INFO : Step 26900/50000 rank6 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0647 | Regression_eta: 0.0286 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0575 | LR: 4.96e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.5620s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank6 [2026-10-29 10:33:37,222] INFO : Step 27000/50000 rank6 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0501 | Regression_eta: 0.0292 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0238 | LR: 4.94e-04 | DataLoad Time: 0.0096s | Model Forward Time: 5.6867s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank6 [2026-10-29 10:33:38,215] INFO : Running validation on rank6 +rank6 [2026-10-29 10:33:38,216] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 10:33:38,216] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 10:34:33,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:34:34,219] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 10:35:00,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 10:35:01,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 10:35:02,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 10:35:03,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 10:35:13,266] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 10:35:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:35:14,237] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 10:35:30,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 10:35:32,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 10:35:33,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 10:35:34,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 10:35:44,521] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 10:35:44,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:35:45,039] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 10:36:00,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 10:36:01,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 10:36:02,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 10:36:03,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 10:36:13,238] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 10:41:28,285] INFO : Step 27100/50000 rank6 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0559 | Regression_eta: 0.0487 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0674 | LR: 4.91e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6738s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank6 [2026-10-29 10:46:22,294] INFO : Step 27200/50000 rank6 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0803 | Regression_eta: 0.0367 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0542 | LR: 4.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8264s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank6 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank6 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0569 | Regression_eta: 0.0378 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0593 | LR: 4.85e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6705s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank6 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank6 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0599 | Regression_eta: 0.0487 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0504 | LR: 4.82e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9750s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank6 [2026-10-29 11:01:35,255] INFO : Step 27500/50000 rank6 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0511 | Regression_eta: 0.0475 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0575 | LR: 4.80e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.3157s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank6 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank6 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0521 | Regression_eta: 0.0347 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0349 | LR: 4.77e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.3332s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank6 [2026-10-29 11:13:33,057] INFO : Step 27700/50000 rank6 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0577 | Regression_eta: 0.0383 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0477 | LR: 4.74e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7949s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank6 [2026-10-29 11:18:30,826] INFO : Step 27800/50000 rank6 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0553 | Regression_eta: 0.0414 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0496 | LR: 4.71e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.0114s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank6 [2026-10-29 11:23:38,362] INFO : Step 27900/50000 rank6 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0647 | Regression_eta: 0.0384 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0571 | LR: 4.68e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8876s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank6 [2026-10-29 11:28:54,872] INFO : Step 28000/50000 rank6 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0621 | Regression_eta: 0.0557 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0964 | LR: 4.66e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.9090s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank6 [2026-10-29 11:28:55,940] INFO : Running validation on rank6 +rank6 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 11:29:47,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:29:48,520] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 11:30:14,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 11:30:15,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 11:30:16,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 11:30:17,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 11:30:27,581] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:30:28,728] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 11:30:44,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 11:30:45,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 11:30:46,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 11:30:47,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 11:30:57,398] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:30:57,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 11:31:14,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 11:31:15,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 11:31:15,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 11:31:16,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 11:31:26,812] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 11:37:01,723] INFO : Step 28100/50000 rank6 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0644 | Regression_eta: 0.0411 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0691 | LR: 4.63e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.5010s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank6 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank6 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0553 | Regression_eta: 0.0367 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0603 | LR: 4.60e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.0743s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank6 [2026-10-29 11:47:09,285] INFO : Step 28300/50000 rank6 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0728 | Regression_eta: 0.0603 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0629 | LR: 4.57e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.1015s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank6 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank6 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1592 | Classification_binary: 0.2529 | Regression_cos_phi: 0.0880 | Regression_energy: 0.1418 | Regression_eta: 0.0735 | Regression_pt: 0.1447 | Regression_sin_phi: 0.1031 | LR: 4.55e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5901s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank6 [2026-10-29 11:56:56,841] INFO : Step 28500/50000 rank6 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0573 | Regression_eta: 0.0523 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0807 | LR: 4.52e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.9894s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank6 [2026-10-29 12:01:46,931] INFO : Step 28600/50000 rank6 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0538 | Regression_eta: 0.0398 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0408 | LR: 4.49e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6735s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank6 [2026-10-29 12:06:39,995] INFO : Step 28700/50000 rank6 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0698 | Regression_eta: 0.0498 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0744 | LR: 4.46e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5053s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank6 [2026-10-29 12:11:31,024] INFO : Step 28800/50000 rank6 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0583 | Regression_eta: 0.0468 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0740 | LR: 4.44e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.9324s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank6 [2026-10-29 12:16:17,098] INFO : Step 28900/50000 rank6 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0543 | Regression_eta: 0.0351 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0642 | LR: 4.41e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7012s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank6 [2026-10-29 12:21:10,134] INFO : Step 29000/50000 rank6 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0613 | Regression_eta: 0.0402 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0636 | LR: 4.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9933s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank6 [2026-10-29 12:21:11,238] INFO : Running validation on rank6 +rank6 [2026-10-29 12:21:11,238] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 12:21:11,238] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 12:22:31,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 12:22:32,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 12:22:33,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 12:22:34,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 12:22:44,911] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:22:45,974] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 12:23:01,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 12:23:02,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 12:23:03,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 12:23:04,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 12:23:14,708] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 12:23:15,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:23:16,447] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 12:23:32,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 12:23:32,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 12:23:33,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 12:23:34,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 12:23:44,655] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank6 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0593 | Regression_eta: 0.0462 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0646 | LR: 4.35e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1724s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank6 [2026-10-29 12:33:43,193] INFO : Step 29200/50000 rank6 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0519 | Regression_eta: 0.0360 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0513 | LR: 4.33e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5662s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank6 [2026-10-29 14:49:26,827] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-29 14:49:26,838] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank6 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-29 14:49:54,089] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank6 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank6 [2026-10-29 14:49:54,124] INFO : Initializing backbone layer 1 +rank6 [2026-10-29 14:49:54,132] INFO : Initializing backbone layer 2 +rank6 [2026-10-29 14:49:54,140] INFO : Initializing backbone layer 3 +rank6 [2026-10-29 14:49:54,148] INFO : Initializing backbone layer 4 +rank6 [2026-10-29 14:49:54,157] INFO : Initializing backbone layer 5 +rank6 [2026-10-29 14:49:54,165] INFO : Backbone initialization took 0.08s +rank6 [2026-10-29 14:49:54,165] INFO : backbone parameters: 7901376 +rank6 [2026-10-29 14:49:54,165] INFO : Initializing output DNNs +rank6 [2026-10-29 14:49:54,202] INFO : Output DNNs initialization took 0.04s +rank6 [2026-10-29 14:49:54,202] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-29 14:49:54,202] INFO : jagged_attention enabled=False +rank6 [2026-10-29 14:49:54,202] INFO : flash_attn_varlen enabled=False +rank6 [2026-10-29 14:49:54,202] INFO : task_queries enabled=False +rank6 [2026-10-29 14:49:54,202] INFO : classification_norm parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : regression_norm parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : classification_query parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : regression_query parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : classification_readout parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : regression_readout parameters: 0 +rank6 [2026-10-29 14:49:54,202] INFO : output_mode=set +rank6 [2026-10-29 14:49:54,203] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-29 14:49:54,203] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_pid parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_pt parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_eta parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-29 14:49:54,203] INFO : nn_energy parameters: 0 +rank6 [2026-10-29 14:49:54,204] INFO : Total MLPF parameters: 12786964 +rank6 [2026-10-29 14:49:54,204] INFO : MLPF __init__ done +rank6 [2026-10-29 14:49:54,204] INFO : Instantiated model +rank6 [2026-10-29 14:49:54,204] INFO : Moving model to device rank=6 +rank6 [2026-10-29 14:49:54,223] INFO : Moved model to device rank=6 +rank6 [2026-10-29 14:49:54,225] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-29 14:49:54,790] INFO : Restoring training from step 29001 +rank6 [2026-10-29 14:49:54,798] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank6 [2026-10-29 14:49:54,798] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-29 14:49:55,363] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-29 14:49:55,515] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank6 [2026-10-29 14:49:55,516] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-29 14:49:55,517] INFO : Trainable parameters: 12786964 +rank6 [2026-10-29 14:49:55,517] INFO : Non-trainable parameters: 23268 +rank6 [2026-10-29 14:49:55,517] INFO : Total parameters: 12810232 +rank6 [2026-10-29 14:49:55,521] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-29 14:49:55,521] INFO : Getting dataloaders +rank6 [2026-10-29 14:49:55,521] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:01,394] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:02,345] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:03,545] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-29 14:50:03,545] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-29 14:50:03,545] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:03,813] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:04,082] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:50:04,337] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-29 14:50:04,337] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-29 14:50:18,020] INFO : Got dataloaders +rank6 [2026-10-29 14:50:18,021] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank6 [2026-10-29 14:50:18,021] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank6 [2026-10-29 14:50:18,021] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank6 [2026-10-29 14:50:18,021] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank6 [2026-10-29 14:50:19,651] INFO : loader split=train rank=6 len=21108 +rank6 [2026-10-29 14:50:19,651] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-29 14:50:19,651] INFO : Starting training from step 29001 to 50000 on rank 6 of 8 +rank6 [2026-10-29 14:50:19,651] INFO : Creating train_iterator +rank6 [2026-10-29 14:50:19,651] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank6 [2026-10-29 14:50:33,634] INFO : Created train_iterator +rank6 [2026-10-29 14:55:12,919] INFO : Step 29100/50000 rank6 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0624 | Regression_eta: 0.0403 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0684 | LR: 4.35e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7161s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank6 [2026-10-29 14:59:46,788] INFO : Step 29200/50000 rank6 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0576 | Regression_eta: 0.0331 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0594 | LR: 4.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3629s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank6 [2026-10-29 15:04:21,833] INFO : Step 29300/50000 rank6 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0585 | Regression_eta: 0.0331 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0416 | LR: 4.30e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0918s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank6 [2026-10-29 15:08:57,030] INFO : Step 29400/50000 rank6 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0472 | Regression_eta: 0.0428 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0758 | LR: 4.27e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2713s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank6 [2026-10-29 15:13:31,010] INFO : Step 29500/50000 rank6 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0605 | Regression_eta: 0.0359 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0592 | LR: 4.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4975s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank6 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank6 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0694 | Regression_eta: 0.0347 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0461 | LR: 4.22e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5926s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank6 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank6 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0615 | Regression_eta: 0.0413 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0623 | LR: 4.19e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6711s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank6 [2026-10-29 15:27:18,562] INFO : Step 29800/50000 rank6 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0587 | Regression_eta: 0.0396 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0830 | LR: 4.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6300s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank6 [2026-10-29 15:31:51,911] INFO : Step 29900/50000 rank6 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0626 | Regression_eta: 0.0342 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0571 | LR: 4.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5839s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank6 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank6 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0476 | Regression_eta: 0.0352 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0423 | LR: 4.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4515s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank6 [2026-10-29 15:36:27,739] INFO : Running validation on rank6 +rank6 [2026-10-29 15:36:27,739] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 15:36:27,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 15:37:19,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 15:37:51,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 15:37:52,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 15:37:53,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 15:37:54,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 15:38:04,293] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:38:04,946] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 15:38:21,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 15:38:22,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 15:38:23,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 15:38:24,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 15:38:34,084] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:38:35,049] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 15:38:50,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 15:38:51,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 15:38:52,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 15:38:53,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 15:39:03,412] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank6 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0678 | Regression_eta: 0.0791 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0666 | LR: 4.08e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7114s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank6 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank6 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0581 | Regression_eta: 0.0403 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0629 | LR: 4.06e-04 | DataLoad Time: 0.0256s | Model Forward Time: 4.7639s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank6 [2026-10-29 15:53:38,867] INFO : Step 30300/50000 rank6 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1234 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0515 | Regression_eta: 0.0515 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0611 | LR: 4.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3373s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank6 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank6 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0486 | Regression_eta: 0.0249 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0600 | LR: 4.00e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4514s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank6 [2026-10-29 16:02:51,486] INFO : Step 30500/50000 rank6 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2533 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0727 | Regression_eta: 0.0514 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0691 | LR: 3.98e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4565s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank6 [2026-10-29 16:07:32,049] INFO : Step 30600/50000 rank6 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0495 | Regression_eta: 0.0255 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0462 | LR: 3.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9371s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank6 [2026-10-29 16:12:18,780] INFO : Step 30700/50000 rank6 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0446 | Regression_eta: 0.0460 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0666 | LR: 3.92e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2820s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank6 [2026-10-29 16:17:03,635] INFO : Step 30800/50000 rank6 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0587 | Regression_eta: 0.0369 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0455 | LR: 3.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0636s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank6 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank6 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0628 | Regression_eta: 0.0392 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0523 | LR: 3.87e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6951s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank6 [2026-10-29 16:26:02,769] INFO : Step 31000/50000 rank6 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0679 | Regression_eta: 0.0342 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0863 | LR: 3.84e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0258s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank6 [2026-10-29 16:26:49,213] INFO : Running validation on rank6 +rank6 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:27:39,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:27:40,304] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 16:28:08,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:28:09,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:28:11,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:28:13,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:28:23,177] INFO : Time taken to make predictions on device 6 is: 0.29 min +rank6 [2026-10-29 16:28:24,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:28:24,724] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 16:28:40,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:28:41,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:28:42,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:28:43,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:28:53,960] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:28:54,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:28:54,678] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 16:29:10,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:29:10,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:29:11,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:29:12,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:29:22,686] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 16:34:37,557] INFO : Step 31100/50000 rank6 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0646 | Regression_eta: 0.0331 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0646 | LR: 3.82e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.5508s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank6 [2026-10-29 16:39:13,624] INFO : Step 31200/50000 rank6 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0434 | Regression_eta: 0.0498 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0855 | LR: 3.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0722s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank6 [2026-10-29 16:43:57,174] INFO : Step 31300/50000 rank6 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0556 | Regression_eta: 0.0444 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0639 | LR: 3.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8832s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank6 [2026-10-29 16:48:30,666] INFO : Step 31400/50000 rank6 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0506 | Regression_eta: 0.0569 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0548 | LR: 3.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7836s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank6 [2026-10-29 16:53:07,752] INFO : Step 31500/50000 rank6 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0509 | Regression_eta: 0.0458 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0657 | LR: 3.71e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4924s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank6 [2026-10-29 16:57:38,607] INFO : Step 31600/50000 rank6 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0439 | Regression_eta: 0.0267 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0410 | LR: 3.69e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1684s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank6 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank6 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0430 | Regression_eta: 0.0372 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0513 | LR: 3.66e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0999s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank6 [2026-10-29 17:06:47,524] INFO : Step 31800/50000 rank6 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0544 | Regression_eta: 0.0382 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0582 | LR: 3.64e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2354s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank6 [2026-10-29 17:11:27,373] INFO : Step 31900/50000 rank6 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0586 | Regression_eta: 0.0477 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0786 | LR: 3.61e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5197s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank6 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank6 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1207 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0626 | Regression_eta: 0.0263 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0541 | LR: 3.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4954s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank6 [2026-10-29 17:16:12,935] INFO : Running validation on rank6 +rank6 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 17:17:03,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:17:04,359] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 17:17:32,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 17:17:33,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 17:17:34,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 17:17:35,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 17:17:45,341] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:17:46,318] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 17:18:02,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 17:18:03,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 17:18:04,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 17:18:05,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 17:18:15,729] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 17:18:15,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:18:16,468] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 17:18:32,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 17:18:33,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 17:18:33,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 17:18:34,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 17:18:44,796] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank6 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0561 | Regression_eta: 0.0337 | Regression_pt: 0.0592 | Regression_sin_phi: 0.1205 | LR: 3.56e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9461s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank6 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank6 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0580 | Regression_eta: 0.0377 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0424 | LR: 3.53e-04 | DataLoad Time: 0.0133s | Model Forward Time: 5.4483s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank6 [2026-10-29 17:33:21,418] INFO : Step 32300/50000 rank6 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1141 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0553 | Regression_eta: 0.0295 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0720 | LR: 3.51e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.9278s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank6 [2026-10-29 17:38:08,351] INFO : Step 32400/50000 rank6 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0525 | Regression_eta: 0.0388 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0728 | LR: 3.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8174s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank6 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank6 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0603 | Regression_eta: 0.0464 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0642 | LR: 3.46e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8573s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank6 [2026-10-29 17:47:36,597] INFO : Step 32600/50000 rank6 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0435 | Regression_eta: 0.0290 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0574 | LR: 3.43e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6207s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank6 [2026-10-29 17:52:17,189] INFO : Step 32700/50000 rank6 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0606 | Regression_eta: 0.0341 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0562 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5869s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank6 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank6 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0544 | Regression_eta: 0.0438 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0573 | LR: 3.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5889s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank6 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank6 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0634 | Regression_eta: 0.0264 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0565 | LR: 3.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4250s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank6 [2026-10-29 18:06:22,611] INFO : Step 33000/50000 rank6 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1058 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0435 | Regression_eta: 0.0323 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0551 | LR: 3.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2117s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank6 [2026-10-29 18:06:23,517] INFO : Running validation on rank6 +rank6 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:07:15,767] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 18:07:43,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:07:45,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:07:46,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:07:47,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:07:57,336] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:07:58,402] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 18:08:14,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:08:15,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:08:16,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:08:17,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:08:27,545] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:08:28,345] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 18:08:44,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:08:44,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:08:45,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:08:46,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:08:56,625] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank6 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1132 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0422 | Regression_eta: 0.0446 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0451 | LR: 3.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2609s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank6 [2026-10-29 18:18:33,981] INFO : Step 33200/50000 rank6 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0461 | Regression_eta: 0.0334 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0521 | LR: 3.28e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4340s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank6 [2026-10-29 18:23:15,421] INFO : Step 33300/50000 rank6 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0466 | Regression_eta: 0.0350 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0582 | LR: 3.26e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6039s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank6 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank6 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0518 | Regression_eta: 0.0386 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0547 | LR: 3.23e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.2630s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank6 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank6 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0508 | Regression_eta: 0.0387 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0446 | LR: 3.21e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3726s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank6 [2026-10-29 18:37:27,752] INFO : Step 33600/50000 rank6 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0458 | Regression_eta: 0.0316 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0454 | LR: 3.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3454s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank6 [2026-10-29 18:42:20,044] INFO : Step 33700/50000 rank6 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0236 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0385 | Regression_eta: 0.0355 | Regression_pt: 0.0423 | Regression_sin_phi: 0.1034 | LR: 3.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3184s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank6 [2026-10-29 18:47:04,986] INFO : Step 33800/50000 rank6 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1403 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0446 | Regression_eta: 0.0327 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0560 | LR: 3.14e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3951s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank6 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank6 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0545 | Regression_eta: 0.0370 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0563 | LR: 3.11e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1926s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank6 [2026-10-29 18:56:35,807] INFO : Step 34000/50000 rank6 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0638 | Regression_eta: 0.0348 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0745 | LR: 3.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1639s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank6 [2026-10-29 18:56:36,807] INFO : Running validation on rank6 +rank6 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:57:29,826] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:57:30,356] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 18:57:54,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:57:55,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:57:56,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:57:57,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:58:07,864] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:58:09,211] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 18:58:25,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:58:26,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:58:27,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:58:28,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:58:38,532] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:58:39,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 18:58:55,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:58:56,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:58:57,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:58:57,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:59:07,959] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 19:04:16,498] INFO : Step 34100/50000 rank6 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1311 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0495 | Regression_eta: 0.0291 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0576 | LR: 3.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8497s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank6 [2026-10-29 19:09:14,637] INFO : Step 34200/50000 rank6 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0462 | Regression_eta: 0.0495 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0639 | LR: 3.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6993s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank6 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank6 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0593 | Regression_eta: 0.0317 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0541 | LR: 3.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3173s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank6 [2026-10-29 19:18:48,204] INFO : Step 34400/50000 rank6 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1153 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0459 | Regression_eta: 0.0440 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0600 | LR: 2.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9242s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank6 [2026-10-29 19:23:49,388] INFO : Step 34500/50000 rank6 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0403 | Regression_eta: 0.0275 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0429 | LR: 2.97e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2310s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank6 [2026-10-29 19:28:34,775] INFO : Step 34600/50000 rank6 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0497 | Regression_eta: 0.0381 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0660 | LR: 2.95e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4747s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank6 [2026-10-29 19:33:35,743] INFO : Step 34700/50000 rank6 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0563 | Regression_eta: 0.0464 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0552 | LR: 2.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1490s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank6 [2026-10-29 19:38:26,589] INFO : Step 34800/50000 rank6 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0465 | Regression_eta: 0.0411 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0396 | LR: 2.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0552s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank6 [2026-10-29 19:43:15,573] INFO : Step 34900/50000 rank6 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0299 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0373 | Regression_eta: 0.0303 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0439 | LR: 2.88e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2758s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank6 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank6 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1080 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0407 | Regression_eta: 0.0399 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0641 | LR: 2.85e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3856s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank6 [2026-10-29 19:48:21,335] INFO : Running validation on rank6 +rank6 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 19:49:15,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:49:15,989] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 19:49:42,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 19:49:43,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 19:49:44,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 19:49:45,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 19:49:55,833] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:49:56,735] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 19:50:12,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 19:50:13,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 19:50:14,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 19:50:15,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 19:50:25,690] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:50:26,883] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 19:50:42,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 19:50:43,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 19:50:44,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 19:50:45,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 19:50:55,480] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 19:56:07,675] INFO : Step 35100/50000 rank6 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1127 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0374 | Regression_eta: 0.0325 | Regression_pt: 0.0384 | Regression_sin_phi: 0.0353 | LR: 2.83e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank6 [2026-10-29 20:01:01,619] INFO : Step 35200/50000 rank6 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1794 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0545 | Regression_eta: 0.0407 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0490 | LR: 2.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6713s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank6 [2026-10-29 20:05:52,156] INFO : Step 35300/50000 rank6 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0508 | Regression_eta: 0.0202 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0382 | LR: 2.79e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1444s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank6 [2026-10-29 20:10:54,438] INFO : Step 35400/50000 rank6 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0478 | Regression_eta: 0.0340 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0595 | LR: 2.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4636s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank6 [2026-10-29 20:15:42,289] INFO : Step 35500/50000 rank6 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0439 | Regression_eta: 0.0258 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0295 | LR: 2.74e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3587s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank6 [2026-10-29 20:20:38,045] INFO : Step 35600/50000 rank6 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1318 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0463 | Regression_eta: 0.0454 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0446 | LR: 2.72e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1897s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank6 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank6 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0583 | Regression_eta: 0.0222 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0454 | LR: 2.70e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1723s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank6 [2026-10-29 20:30:22,090] INFO : Step 35800/50000 rank6 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0535 | Regression_eta: 0.0329 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0439 | LR: 2.68e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2788s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank6 [2026-10-29 20:35:20,188] INFO : Step 35900/50000 rank6 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0532 | Regression_eta: 0.0296 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0683 | LR: 2.65e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6679s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank6 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank6 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0305 | Regression_energy: 0.0482 | Regression_eta: 0.0244 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0724 | LR: 2.63e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2574s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank6 [2026-10-29 20:40:17,059] INFO : Running validation on rank6 +rank6 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 20:41:08,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:41:08,960] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 20:41:36,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 20:41:37,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 20:41:38,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 20:41:39,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 20:41:49,227] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:41:50,341] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 20:42:06,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 20:42:07,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 20:42:08,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 20:42:09,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 20:42:19,508] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:42:21,034] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 20:42:36,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 20:42:37,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 20:42:38,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 20:42:39,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 20:42:49,194] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 20:47:46,990] INFO : Step 36100/50000 rank6 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0488 | Regression_eta: 0.0352 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0694 | LR: 2.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4501s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank6 [2026-10-29 20:52:22,842] INFO : Step 36200/50000 rank6 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0453 | Regression_eta: 0.0338 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0546 | LR: 2.59e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3970s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank6 [2026-10-29 20:56:59,586] INFO : Step 36300/50000 rank6 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0513 | Regression_eta: 0.0262 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0467 | LR: 2.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2283s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank6 [2026-10-29 21:01:38,352] INFO : Step 36400/50000 rank6 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.0984 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0339 | Regression_eta: 0.0195 | Regression_pt: 0.0363 | Regression_sin_phi: 0.0266 | LR: 2.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8766s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank6 [2026-10-29 21:06:14,245] INFO : Step 36500/50000 rank6 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1250 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0488 | Regression_eta: 0.0279 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0503 | LR: 2.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4221s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank6 [2026-10-29 21:10:49,732] INFO : Step 36600/50000 rank6 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0286 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0558 | Regression_eta: 0.0420 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0375 | LR: 2.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2770s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank6 [2026-10-29 21:15:35,109] INFO : Step 36700/50000 rank6 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1126 | Regression_cos_phi: 0.0343 | Regression_energy: 0.0411 | Regression_eta: 0.0269 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0240 | LR: 2.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2709s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank6 [2026-10-29 21:20:28,003] INFO : Step 36800/50000 rank6 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0720 | Regression_eta: 0.0489 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0706 | LR: 2.46e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3977s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank6 [2026-10-29 21:25:23,682] INFO : Step 36900/50000 rank6 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0572 | Regression_eta: 0.0280 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0404 | LR: 2.44e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9354s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank6 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank6 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0451 | Regression_eta: 0.0326 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0579 | LR: 2.42e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9282s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank6 [2026-10-29 21:30:17,323] INFO : Running validation on rank6 +rank6 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:31:08,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:31:08,775] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 21:31:35,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:31:36,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:31:37,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:31:38,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:31:48,467] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:31:49,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:31:49,596] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 21:32:06,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:32:07,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:32:08,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:32:09,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:32:19,571] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:32:20,812] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 21:32:36,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:32:37,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:32:38,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:32:39,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:32:49,297] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 21:37:56,198] INFO : Step 37100/50000 rank6 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0444 | Regression_eta: 0.0254 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0532 | LR: 2.40e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6910s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank6 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank6 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0399 | Regression_eta: 0.0256 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0426 | LR: 2.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6958s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank6 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank6 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0517 | Regression_eta: 0.0285 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0422 | LR: 2.36e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5587s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank6 [2026-10-29 21:52:56,910] INFO : Step 37400/50000 rank6 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1060 | Regression_cos_phi: 0.0333 | Regression_energy: 0.0574 | Regression_eta: 0.0234 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0370 | LR: 2.34e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6112s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank6 [2026-10-29 21:57:43,877] INFO : Step 37500/50000 rank6 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0382 | Regression_eta: 0.0339 | Regression_pt: 0.0389 | Regression_sin_phi: 0.0454 | LR: 2.32e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7096s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank6 [2026-10-29 22:02:29,840] INFO : Step 37600/50000 rank6 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1143 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0534 | Regression_eta: 0.0298 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0411 | LR: 2.30e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2475s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank6 [2026-10-29 22:07:05,609] INFO : Step 37700/50000 rank6 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1129 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0460 | Regression_eta: 0.0266 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0522 | LR: 2.28e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1549s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank6 [2026-10-29 22:11:38,775] INFO : Step 37800/50000 rank6 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1186 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0452 | Regression_eta: 0.0318 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0478 | LR: 2.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2576s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank6 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank6 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0508 | Regression_eta: 0.0399 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0454 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1265s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank6 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank6 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0554 | Regression_eta: 0.0398 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0455 | LR: 2.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2877s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank6 [2026-10-29 22:20:57,876] INFO : Running validation on rank6 +rank6 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 22:21:48,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:21:48,922] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 22:22:15,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 22:22:16,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 22:22:17,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 22:22:18,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 22:22:28,432] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 22:22:28,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:22:29,427] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 22:22:45,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 22:22:46,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 22:22:47,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 22:22:47,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 22:22:57,994] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:22:59,393] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 22:23:15,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 22:23:16,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 22:23:16,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 22:23:17,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 22:23:27,743] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank6 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0625 | Regression_eta: 0.0379 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0567 | LR: 2.20e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2300s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank6 [2026-10-29 22:33:07,240] INFO : Step 38200/50000 rank6 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0622 | Regression_eta: 0.0309 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0397 | LR: 2.18e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7610s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank6 [2026-10-29 22:37:42,243] INFO : Step 38300/50000 rank6 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1111 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0422 | Regression_eta: 0.0246 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0353 | LR: 2.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5946s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank6 [2026-10-29 22:42:17,609] INFO : Step 38400/50000 rank6 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1058 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0635 | Regression_eta: 0.0364 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0814 | LR: 2.14e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4886s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank6 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank6 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1109 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0450 | Regression_eta: 0.0308 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0421 | LR: 2.12e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5217s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank6 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank6 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0465 | Regression_eta: 0.0388 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0585 | LR: 2.11e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7265s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank6 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank6 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0445 | Regression_eta: 0.0274 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0424 | LR: 2.09e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1555s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank6 [2026-10-29 23:02:19,419] INFO : Step 38800/50000 rank6 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0551 | Regression_eta: 0.0319 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0481 | LR: 2.07e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3987s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank6 [2026-10-29 23:07:29,433] INFO : Step 38900/50000 rank6 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1396 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0387 | Regression_eta: 0.0310 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0690 | LR: 2.05e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.8886s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank6 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank6 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0475 | Regression_eta: 0.0330 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0488 | LR: 2.03e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4687s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank6 [2026-10-29 23:12:38,866] INFO : Running validation on rank6 +rank6 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 23:13:33,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:13:34,232] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-29 23:14:06,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 23:14:07,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 23:14:08,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 23:14:09,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 23:14:19,528] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:14:20,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:14:21,119] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-29 23:14:36,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 23:14:38,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 23:14:38,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 23:14:39,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 23:14:49,959] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:14:50,900] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-29 23:15:06,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 23:15:07,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 23:15:08,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 23:15:09,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 23:15:19,148] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank6 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0505 | Regression_eta: 0.0392 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0464 | LR: 2.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9862s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank6 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank6 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0427 | Regression_eta: 0.0609 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0778 | LR: 2.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5908s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank6 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank6 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0457 | Regression_eta: 0.0361 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0589 | LR: 1.98e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6311s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank6 [2026-10-29 23:34:45,046] INFO : Step 39400/50000 rank6 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0497 | Regression_eta: 0.0272 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0355 | LR: 1.96e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.6884s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank6 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank6 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0509 | Regression_eta: 0.0329 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0558 | LR: 1.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4401s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank6 [2026-10-29 23:43:57,879] INFO : Step 39600/50000 rank6 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0513 | Regression_eta: 0.0363 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0678 | LR: 1.93e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4684s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank6 [2026-10-29 23:48:42,957] INFO : Step 39700/50000 rank6 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0533 | Regression_eta: 0.0277 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0504 | LR: 1.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4045s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank6 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank6 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0594 | Regression_eta: 0.0389 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0610 | LR: 1.89e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5379s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank6 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank6 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0448 | Regression_eta: 0.0415 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0606 | LR: 1.88e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7669s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank6 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank6 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0470 | Regression_eta: 0.0352 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0382 | LR: 1.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9273s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank6 [2026-10-30 00:05:56,736] INFO : Running validation on rank6 +rank6 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:06:50,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:06:51,354] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 00:07:17,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:07:18,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:07:19,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:07:20,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:07:30,952] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:07:31,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 00:07:47,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:07:48,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:07:49,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:07:50,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:08:00,959] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:08:01,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 00:08:18,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 00:08:18,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 00:08:19,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 00:08:20,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 00:08:30,610] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank6 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0488 | Regression_eta: 0.0271 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0313 | LR: 1.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6421s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank6 [2026-10-30 00:18:43,887] INFO : Step 40200/50000 rank6 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0460 | Regression_eta: 0.0224 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0426 | LR: 1.83e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4438s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank6 [2026-10-30 00:23:35,703] INFO : Step 40300/50000 rank6 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0530 | Regression_eta: 0.0600 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0676 | LR: 1.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5331s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank6 [2026-10-30 00:28:38,368] INFO : Step 40400/50000 rank6 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0465 | Regression_eta: 0.0375 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0653 | LR: 1.79e-04 | DataLoad Time: 0.0115s | Model Forward Time: 4.2155s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank6 [2026-10-30 00:33:25,152] INFO : Step 40500/50000 rank6 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1023 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0513 | Regression_eta: 0.0213 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0318 | LR: 1.78e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2279s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank6 [2026-10-30 00:38:07,832] INFO : Step 40600/50000 rank6 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0277 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0493 | Regression_eta: 0.0357 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0522 | LR: 1.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2961s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank6 [2026-10-30 00:42:54,583] INFO : Step 40700/50000 rank6 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0275 | Regression_energy: 0.0418 | Regression_eta: 0.0255 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0352 | LR: 1.75e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2533s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank6 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank6 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1002 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0437 | Regression_eta: 0.0239 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0278 | LR: 1.73e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8888s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank6 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank6 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1042 | Regression_cos_phi: 0.0386 | Regression_energy: 0.0450 | Regression_eta: 0.0210 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0351 | LR: 1.72e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7072s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank6 [2026-10-30 00:57:38,929] INFO : Step 41000/50000 rank6 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0513 | Regression_eta: 0.0351 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0418 | LR: 1.70e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3873s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank6 [2026-10-30 00:57:39,931] INFO : Running validation on rank6 +rank6 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:58:31,731] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:58:32,294] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 00:58:59,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:59:00,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:59:01,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:59:02,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:59:12,412] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:59:13,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:59:13,671] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 00:59:29,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:59:30,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:59:31,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:59:32,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:59:42,543] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:59:43,890] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 00:59:59,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:00:00,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:00:01,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:00:02,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:00:12,379] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 01:05:10,996] INFO : Step 41100/50000 rank6 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0381 | Regression_eta: 0.0284 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0456 | LR: 1.69e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7649s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank6 [2026-10-30 01:10:09,652] INFO : Step 41200/50000 rank6 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1091 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0499 | Regression_eta: 0.0476 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0812 | LR: 1.67e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6348s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank6 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank6 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0449 | Regression_eta: 0.0331 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0374 | LR: 1.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2627s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank6 [2026-10-30 01:19:55,521] INFO : Step 41400/50000 rank6 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0515 | Regression_eta: 0.0314 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0439 | LR: 1.64e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.6704s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank6 [2026-10-30 01:24:55,601] INFO : Step 41500/50000 rank6 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0535 | Regression_eta: 0.0284 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0417 | LR: 1.63e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4499s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank6 [2026-10-30 01:29:39,759] INFO : Step 41600/50000 rank6 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0435 | Regression_eta: 0.0208 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0433 | LR: 1.61e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0143s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank6 [2026-10-30 01:34:19,968] INFO : Step 41700/50000 rank6 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0430 | Regression_eta: 0.0321 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0388 | LR: 1.60e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7030s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank6 [2026-10-30 01:39:10,938] INFO : Step 41800/50000 rank6 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0517 | Regression_eta: 0.0443 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0659 | LR: 1.58e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8786s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank6 [2026-10-30 01:44:06,919] INFO : Step 41900/50000 rank6 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0417 | Regression_eta: 0.0430 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0423 | LR: 1.57e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2467s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank6 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank6 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0432 | Regression_eta: 0.0363 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0494 | LR: 1.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2685s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank6 [2026-10-30 01:49:03,869] INFO : Running validation on rank6 +rank6 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 01:49:57,369] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 01:50:24,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 01:50:26,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 01:50:27,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 01:50:28,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 01:50:38,132] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-30 01:50:38,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:50:39,487] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 01:50:55,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 01:50:56,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 01:50:57,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 01:50:58,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 01:51:08,676] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:51:10,101] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 01:51:25,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:51:26,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:51:27,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:51:28,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:51:38,356] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank6 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0305 | Regression_energy: 0.0386 | Regression_eta: 0.0304 | Regression_pt: 0.0390 | Regression_sin_phi: 0.0341 | LR: 1.54e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0034s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank6 [2026-10-30 02:01:47,468] INFO : Step 42200/50000 rank6 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0457 | Regression_eta: 0.0239 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0506 | LR: 1.53e-04 | DataLoad Time: 0.0160s | Model Forward Time: 7.2728s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank6 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank6 [2026-10-30 02:02:38,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-30 02:06:41,297] INFO : Step 42300/50000 rank6 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0291 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0489 | Regression_eta: 0.0269 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0277 | LR: 1.52e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8839s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank6 [2026-10-30 02:11:46,437] INFO : Step 42400/50000 rank6 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0272 | Classification_binary: 0.0979 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0371 | Regression_eta: 0.0322 | Regression_pt: 0.0383 | Regression_sin_phi: 0.0358 | LR: 1.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.1596s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank6 [2026-10-30 02:16:38,996] INFO : Step 42500/50000 rank6 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0433 | Regression_eta: 0.0328 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0492 | LR: 1.49e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8606s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank6 [2026-10-30 02:21:27,118] INFO : Step 42600/50000 rank6 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0360 | Regression_eta: 0.0279 | Regression_pt: 0.0370 | Regression_sin_phi: 0.0362 | LR: 1.48e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.3558s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank6 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank6 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0273 | Classification_binary: 0.1243 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0451 | Regression_eta: 0.0240 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0378 | LR: 1.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6230s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank6 [2026-10-30 02:31:38,796] INFO : Step 42800/50000 rank6 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0485 | Regression_eta: 0.0229 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0395 | LR: 1.45e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6419s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank6 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank6 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0361 | Regression_eta: 0.0245 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0619 | LR: 1.44e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8534s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank6 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank6 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1009 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0337 | Regression_eta: 0.0276 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0349 | LR: 1.43e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5943s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank6 [2026-10-30 02:41:14,879] INFO : Running validation on rank6 +rank6 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 02:42:10,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 02:42:38,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 02:42:39,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 02:42:40,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 02:42:41,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 02:42:51,186] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 02:42:51,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:42:52,441] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 02:43:08,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 02:43:09,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 02:43:10,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 02:43:11,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 02:43:21,691] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 02:43:38,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 02:43:39,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 02:43:40,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 02:43:41,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 02:43:51,324] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 02:48:51,376] INFO : Step 43100/50000 rank6 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0456 | Regression_eta: 0.0446 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0651 | LR: 1.42e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9468s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank6 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank6 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0543 | Regression_eta: 0.0384 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0407 | LR: 1.40e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5639s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank6 [2026-10-30 02:58:07,501] INFO : Step 43300/50000 rank6 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0436 | Regression_eta: 0.0382 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0474 | LR: 1.39e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0486s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank6 [2026-10-30 03:02:42,808] INFO : Step 43400/50000 rank6 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1002 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0402 | Regression_eta: 0.0211 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0443 | LR: 1.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4742s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank6 [2026-10-30 03:07:16,795] INFO : Step 43500/50000 rank6 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0402 | Regression_eta: 0.0354 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0415 | LR: 1.37e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9342s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank6 [2026-10-30 03:12:00,121] INFO : Step 43600/50000 rank6 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0613 | Regression_eta: 0.0274 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0435 | LR: 1.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8414s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank6 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank6 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0387 | Regression_eta: 0.0253 | Regression_pt: 0.0397 | Regression_sin_phi: 0.0387 | LR: 1.35e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1357s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank6 [2026-10-30 03:21:10,514] INFO : Step 43800/50000 rank6 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0232 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0281 | Regression_energy: 0.0411 | Regression_eta: 0.0225 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0336 | LR: 1.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2539s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank6 [2026-10-30 03:26:01,000] INFO : Step 43900/50000 rank6 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0478 | Regression_eta: 0.0484 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0708 | LR: 1.33e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9694s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank6 [2026-10-30 03:30:36,933] INFO : Step 44000/50000 rank6 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0224 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0336 | Regression_energy: 0.0467 | Regression_eta: 0.0254 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0544 | LR: 1.32e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8170s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank6 [2026-10-30 03:30:37,938] INFO : Running validation on rank6 +rank6 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 03:31:28,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:31:29,164] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 03:31:57,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 03:31:58,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 03:31:59,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 03:32:00,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 03:32:10,907] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 03:32:27,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 03:32:28,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 03:32:29,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 03:32:30,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 03:32:40,683] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:32:41,628] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 03:32:57,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 03:32:58,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 03:32:59,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 03:33:00,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 03:33:10,033] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 03:38:15,295] INFO : Step 44100/50000 rank6 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0466 | Regression_eta: 0.0342 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0465 | LR: 1.31e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7497s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank6 [2026-10-30 03:43:04,521] INFO : Step 44200/50000 rank6 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0336 | Regression_energy: 0.0415 | Regression_eta: 0.0231 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0374 | LR: 1.30e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4213s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank6 [2026-10-30 03:47:46,464] INFO : Step 44300/50000 rank6 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0501 | Regression_eta: 0.0448 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0640 | LR: 1.29e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4323s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank6 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank6 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0278 | Classification_binary: 0.0909 | Regression_cos_phi: 0.0309 | Regression_energy: 0.0372 | Regression_eta: 0.0247 | Regression_pt: 0.0383 | Regression_sin_phi: 0.0367 | LR: 1.28e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3358s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank6 [2026-10-30 03:57:11,971] INFO : Step 44500/50000 rank6 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0460 | Regression_eta: 0.0418 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0968 | LR: 1.27e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3161s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank6 [2026-10-30 04:01:48,197] INFO : Step 44600/50000 rank6 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1568 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0512 | Regression_eta: 0.0427 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0682 | LR: 1.26e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank6 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank6 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.0965 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0526 | Regression_eta: 0.0201 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0243 | LR: 1.25e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.6208s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank6 [2026-10-30 04:11:07,219] INFO : Step 44800/50000 rank6 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0344 | Regression_eta: 0.0261 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0397 | LR: 1.24e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5863s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank6 [2026-10-30 04:15:54,537] INFO : Step 44900/50000 rank6 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0486 | Regression_eta: 0.0248 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0335 | LR: 1.23e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6735s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank6 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank6 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0259 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0523 | Regression_eta: 0.0273 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0549 | LR: 1.22e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7886s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank6 [2026-10-30 04:20:37,109] INFO : Running validation on rank6 +rank6 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 04:21:27,992] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:21:28,558] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 04:21:56,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 04:21:57,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 04:21:58,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 04:21:59,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 04:22:09,983] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:22:10,892] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 04:22:26,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 04:22:27,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 04:22:28,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 04:22:29,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 04:22:39,580] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 04:22:40,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:22:41,054] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 04:22:56,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 04:22:57,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 04:22:58,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 04:22:59,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 04:23:09,494] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank6 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0296 | Classification_binary: 0.1031 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0355 | Regression_eta: 0.0315 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0324 | LR: 1.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2109s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank6 [2026-10-30 04:32:57,432] INFO : Step 45200/50000 rank6 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0477 | Regression_eta: 0.0235 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0396 | LR: 1.20e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank6 [2026-10-30 04:37:52,085] INFO : Step 45300/50000 rank6 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0462 | Regression_eta: 0.0323 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0431 | LR: 1.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8153s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank6 [2026-10-30 04:42:38,427] INFO : Step 45400/50000 rank6 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0203 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0293 | Regression_energy: 0.0674 | Regression_eta: 0.0211 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0294 | LR: 1.19e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4025s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank6 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank6 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0353 | Regression_eta: 0.0227 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0457 | LR: 1.18e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3873s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank6 [2026-10-30 04:52:16,190] INFO : Step 45600/50000 rank6 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1244 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0449 | Regression_eta: 0.0183 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0309 | LR: 1.17e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4794s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank6 [2026-10-30 04:57:07,844] INFO : Step 45700/50000 rank6 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0504 | Regression_eta: 0.0370 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0672 | LR: 1.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4879s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank6 [2026-10-30 05:01:56,081] INFO : Step 45800/50000 rank6 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0310 | Classification_binary: 0.0900 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0456 | Regression_eta: 0.0323 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0758 | LR: 1.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6463s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank6 [2026-10-30 05:06:42,061] INFO : Step 45900/50000 rank6 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0432 | Regression_eta: 0.0468 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0637 | LR: 1.15e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2865s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank6 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank6 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0243 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0440 | Regression_eta: 0.0267 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0344 | LR: 1.14e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.4342s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank6 [2026-10-30 05:11:37,038] INFO : Running validation on rank6 +rank6 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 05:12:27,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:12:27,993] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 05:12:53,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 05:12:54,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 05:12:55,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 05:12:56,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 05:13:06,320] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 05:13:23,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 05:13:24,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 05:13:25,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 05:13:26,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 05:13:36,188] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 05:13:36,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 05:13:52,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 05:13:53,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 05:13:54,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 05:13:55,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 05:14:05,576] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 05:19:00,536] INFO : Step 46100/50000 rank6 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.0876 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0417 | Regression_eta: 0.0234 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0294 | LR: 1.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5905s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank6 [2026-10-30 05:23:34,446] INFO : Step 46200/50000 rank6 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0232 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0482 | Regression_eta: 0.0248 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0557 | LR: 1.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7929s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank6 [2026-10-30 05:28:08,608] INFO : Step 46300/50000 rank6 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0607 | Regression_eta: 0.0303 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0478 | LR: 1.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9770s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank6 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank6 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0455 | Regression_eta: 0.0179 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0498 | LR: 1.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7883s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank6 [2026-10-30 05:37:21,832] INFO : Step 46500/50000 rank6 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0535 | Regression_eta: 0.0244 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0451 | LR: 1.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8422s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank6 [2026-10-30 05:41:57,345] INFO : Step 46600/50000 rank6 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0474 | Regression_eta: 0.0300 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0440 | LR: 1.10e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7860s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank6 [2026-10-30 05:46:32,412] INFO : Step 46700/50000 rank6 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0614 | Regression_eta: 0.0246 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0428 | LR: 1.10e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6533s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank6 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank6 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1365 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0353 | Regression_eta: 0.0316 | Regression_pt: 0.0364 | Regression_sin_phi: 0.0390 | LR: 1.09e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7022s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank6 [2026-10-30 05:55:41,597] INFO : Step 46900/50000 rank6 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0418 | Regression_eta: 0.0367 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0434 | LR: 1.09e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7520s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank6 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank6 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0449 | Regression_eta: 0.0341 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0539 | LR: 1.08e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8054s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank6 [2026-10-30 06:00:21,750] INFO : Running validation on rank6 +rank6 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 06:00:21,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 06:01:12,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:01:12,841] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 06:01:37,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 06:01:38,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 06:01:39,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 06:01:40,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 06:01:50,588] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:01:51,691] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 06:02:07,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 06:02:08,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 06:02:09,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 06:02:10,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 06:02:20,580] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:02:21,869] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 06:02:37,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 06:02:38,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 06:02:39,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 06:02:40,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 06:02:50,096] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 06:07:51,194] INFO : Step 47100/50000 rank6 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.0976 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0397 | Regression_eta: 0.0369 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0435 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6247s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank6 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank6 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0343 | Regression_eta: 0.0562 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0743 | LR: 1.07e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9897s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank6 [2026-10-30 06:17:04,151] INFO : Step 47300/50000 rank6 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0449 | Regression_eta: 0.0323 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0466 | LR: 1.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0314s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank6 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank6 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0525 | Regression_eta: 0.0285 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0518 | LR: 1.06e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4517s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank6 [2026-10-30 06:26:15,800] INFO : Step 47500/50000 rank6 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.0935 | Regression_cos_phi: 0.0289 | Regression_energy: 0.0382 | Regression_eta: 0.0245 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0278 | LR: 1.06e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6759s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank6 [2026-10-30 06:30:50,119] INFO : Step 47600/50000 rank6 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0223 | Classification_binary: 0.0943 | Regression_cos_phi: 0.0275 | Regression_energy: 0.0389 | Regression_eta: 0.0193 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0259 | LR: 1.05e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9481s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank6 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank6 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0565 | Regression_eta: 0.0387 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0652 | LR: 1.05e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9100s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank6 [2026-10-30 06:40:00,408] INFO : Step 47800/50000 rank6 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1159 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0421 | Regression_eta: 0.0525 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0694 | LR: 1.04e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7798s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank6 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank6 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0502 | Regression_eta: 0.0234 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0353 | LR: 1.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8493s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank6 [2026-10-30 06:49:09,636] INFO : Step 48000/50000 rank6 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0399 | Regression_eta: 0.0282 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0507 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8815s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank6 [2026-10-30 06:49:10,727] INFO : Running validation on rank6 +rank6 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 06:50:01,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:50:01,897] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 06:50:27,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 06:50:28,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 06:50:29,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 06:50:30,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 06:50:40,302] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:50:41,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:50:41,778] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 06:50:57,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 06:50:58,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 06:50:59,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 06:51:00,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 06:51:10,612] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:51:11,430] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 06:51:27,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 06:51:28,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 06:51:29,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 06:51:29,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 06:51:39,825] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank6 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0433 | Regression_eta: 0.0278 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0361 | LR: 1.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6274s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank6 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank6 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0285 | Regression_energy: 0.0331 | Regression_eta: 0.0205 | Regression_pt: 0.0342 | Regression_sin_phi: 0.0362 | LR: 1.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3153s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank6 [2026-10-30 07:05:53,464] INFO : Step 48300/50000 rank6 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0480 | Regression_eta: 0.0316 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0434 | LR: 1.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1166s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank6 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank6 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0400 | Regression_eta: 0.0363 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0445 | LR: 1.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8683s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank6 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank6 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0476 | Regression_eta: 0.0207 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0387 | LR: 1.02e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.9618s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank6 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank6 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0474 | Regression_eta: 0.0361 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0421 | LR: 1.02e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8075s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank6 [2026-10-30 07:24:30,057] INFO : Step 48700/50000 rank6 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0581 | Regression_eta: 0.0336 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0570 | LR: 1.02e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.9982s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank6 [2026-10-30 07:29:10,217] INFO : Step 48800/50000 rank6 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1098 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0516 | Regression_eta: 0.0334 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0494 | LR: 1.01e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9234s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank6 [2026-10-30 07:33:48,296] INFO : Step 48900/50000 rank6 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0425 | Regression_eta: 0.0235 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0448 | LR: 1.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7897s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank6 [2026-10-30 07:38:22,688] INFO : Step 49000/50000 rank6 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0411 | Regression_eta: 0.0284 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0524 | LR: 1.01e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.8900s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank6 [2026-10-30 07:38:23,699] INFO : Running validation on rank6 +rank6 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:39:14,628] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 07:39:39,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 07:39:40,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 07:39:41,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 07:39:42,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 07:39:52,667] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 07:39:53,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:39:53,779] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 07:40:09,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 07:40:10,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 07:40:11,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 07:40:12,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 07:40:22,881] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:40:24,012] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 07:40:39,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 07:40:40,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 07:40:41,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 07:40:42,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 07:40:52,195] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank6 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0524 | Regression_eta: 0.0389 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0519 | LR: 1.01e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.6559s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank6 [2026-10-30 07:50:30,114] INFO : Step 49200/50000 rank6 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0503 | Regression_eta: 0.0280 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0380 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5148s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank6 [2026-10-30 07:55:07,371] INFO : Step 49300/50000 rank6 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0487 | Regression_eta: 0.0317 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0409 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8681s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank6 [2026-10-30 07:59:43,351] INFO : Step 49400/50000 rank6 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1260 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0482 | Regression_eta: 0.0248 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0369 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8895s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank6 [2026-10-30 08:04:24,758] INFO : Step 49500/50000 rank6 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.0823 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0508 | Regression_eta: 0.0227 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0281 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3347s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank6 [2026-10-30 08:08:58,923] INFO : Step 49600/50000 rank6 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0406 | Regression_eta: 0.0286 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0487 | LR: 1.00e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8767s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank6 [2026-10-30 08:13:36,932] INFO : Step 49700/50000 rank6 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1257 | Regression_cos_phi: 0.0320 | Regression_energy: 0.0422 | Regression_eta: 0.0227 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0256 | LR: 1.00e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7041s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank6 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank6 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0454 | Regression_eta: 0.0299 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0500 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5269s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank6 [2026-10-30 08:22:53,416] INFO : Step 49900/50000 rank6 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0441 | Regression_eta: 0.0341 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0416 | LR: 1.00e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2023s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank6 [2026-10-30 08:27:37,250] INFO : Step 50000/50000 rank6 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0250 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0416 | Regression_eta: 0.0239 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0482 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3469s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank6 [2026-10-30 08:27:38,187] INFO : Running validation on rank6 +rank6 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 08:28:29,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 08:28:29,586] INFO : Running predictions on clic_edm_ttbar_hits +rank6 [2026-10-30 08:29:00,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 08:29:01,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 08:29:02,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 08:29:03,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 08:29:13,143] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 08:29:13,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 08:29:14,403] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank6 [2026-10-30 08:29:30,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 08:29:32,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 08:29:33,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 08:29:35,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 08:29:45,403] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 08:29:46,217] INFO : Running predictions on clic_edm_qq_hits +rank6 [2026-10-30 08:30:02,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 08:30:02,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 08:30:03,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 08:30:04,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 08:30:14,717] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 08:30:15,206] INFO : Training completed. Total time on device 6: 1059.926min diff --git a/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.7 b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..a0e13cb67f615d8af250021325beac371c57979b --- /dev/null +++ b/clic/hits/v3.2.0/clic_hits_heptv2_seed12345_20261028_111202_012645/train.log.7 @@ -0,0 +1,2654 @@ +rank7 [2026-10-28 11:12:32,871] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-28 11:12:32,872] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-28 11:12:57,191] INFO : Instantiating model +rank7 [2026-10-28 11:12:57,191] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-28 11:12:57,191] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-28 11:12:57,209] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-28 11:12:57,209] INFO : nn0 parameters: 171776 +rank7 [2026-10-28 11:12:57,210] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-28 11:12:57,210] INFO : Initializing backbone layer 0 +rank7 [2026-10-28 11:12:57,245] INFO : Initializing backbone layer 1 +rank7 [2026-10-28 11:12:57,254] INFO : Initializing backbone layer 2 +rank7 [2026-10-28 11:12:57,263] INFO : Initializing backbone layer 3 +rank7 [2026-10-28 11:12:57,272] INFO : Initializing backbone layer 4 +rank7 [2026-10-28 11:12:57,280] INFO : Initializing backbone layer 5 +rank7 [2026-10-28 11:12:57,289] INFO : Backbone initialization took 0.08s +rank7 [2026-10-28 11:12:57,289] INFO : backbone parameters: 7901376 +rank7 [2026-10-28 11:12:57,290] INFO : Initializing output DNNs +rank7 [2026-10-28 11:12:57,328] INFO : Output DNNs initialization took 0.04s +rank7 [2026-10-28 11:12:57,328] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-28 11:12:57,328] INFO : jagged_attention enabled=False +rank7 [2026-10-28 11:12:57,328] INFO : flash_attn_varlen enabled=False +rank7 [2026-10-28 11:12:57,328] INFO : task_queries enabled=False +rank7 [2026-10-28 11:12:57,329] INFO : classification_norm parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : regression_norm parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : classification_query parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : regression_query parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : classification_readout parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : regression_readout parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : output_mode=set +rank7 [2026-10-28 11:12:57,329] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-28 11:12:57,329] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_pid parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_pt parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_eta parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-28 11:12:57,329] INFO : nn_energy parameters: 0 +rank7 [2026-10-28 11:12:57,330] INFO : Total MLPF parameters: 12786964 +rank7 [2026-10-28 11:12:57,330] INFO : MLPF __init__ done +rank7 [2026-10-28 11:12:57,330] INFO : Instantiated model +rank7 [2026-10-28 11:12:57,330] INFO : Moving model to device rank=7 +rank7 [2026-10-28 11:12:57,368] INFO : Moved model to device rank=7 +rank7 [2026-10-28 11:12:57,369] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-28 11:12:57,370] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-28 11:12:57,694] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-28 11:12:57,855] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank7 [2026-10-28 11:12:57,855] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-28 11:12:57,856] INFO : Trainable parameters: 12786964 +rank7 [2026-10-28 11:12:57,856] INFO : Non-trainable parameters: 23268 +rank7 [2026-10-28 11:12:57,856] INFO : Total parameters: 12810232 +rank7 [2026-10-28 11:12:57,859] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-28 11:12:57,859] INFO : Getting dataloaders +rank7 [2026-10-28 11:12:57,859] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:03,690] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:04,754] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,001] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-28 11:13:06,001] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-28 11:13:06,001] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,273] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,533] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,797] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-28 11:13:06,797] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-28 11:13:20,650] INFO : Got dataloaders +rank7 [2026-10-28 11:13:20,650] INFO : loader split=train rank=7 len=21108 +rank7 [2026-10-28 11:13:20,650] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-28 11:13:20,650] INFO : Starting training from step 1 to 50000 on rank 7 of 8 +rank7 [2026-10-28 11:13:20,651] INFO : Creating train_iterator +rank7 [2026-10-28 11:13:20,651] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-28 11:13:35,191] INFO : Created train_iterator +rank7 [2026-10-28 11:18:13,949] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5540849350392818, 'Classification': 0.9389768500626087, 'Regression_pt': 1.03104786247015, 'Regression_eta': 0.01951152349822223, 'Regression_sin_phi': 0.007536812543112319, 'Regression_cos_phi': 0.007328496277623344, 'Regression_energy': 1.0578305500745773}, weights={'Classification_binary': 1.0, 'Classification': 0.5900943279266357, 'Regression_pt': 0.5373998284339905, 'Regression_eta': 28.397830963134766, 'Regression_sin_phi': 73.51714324951172, 'Regression_cos_phi': 75.60690307617188, 'Regression_energy': 0.523793637752533} +rank7 [2026-10-28 11:18:16,511] INFO : Step 100/50000 rank7 | Train Loss: 4.8722 | Train Loss Unweighted: 2.9326 | Train Loss Components: Auxiliary: 0.8908 | Cardinality: 0.6976 | Classification: 0.7630 | Classification_binary: 0.4724 | HitGrouping: 0.3513 | Regression_cos_phi: 0.0058 | Regression_energy: 0.8532 | Regression_eta: 0.0132 | Regression_pt: 0.8185 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8507 | Classification_binary: 0.4400 | Regression_cos_phi: 0.0061 | Regression_energy: 0.9780 | Regression_eta: 0.0139 | Regression_pt: 0.9578 | Regression_sin_phi: 0.0054 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.6183s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6780 | max_loss: 0.2285 | max_backward: 1.4415 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0141 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4775.6 | uss_MB: 3472.0 | child_rss_MB: 6366.9 | allocated_MB: 447.4 | reserved_MB: 24618.0 | inactive_split_MB: 1253.2 +rank7 [2026-10-28 11:22:47,963] INFO : Step 200/50000 rank7 | Train Loss: 4.3065 | Train Loss Unweighted: 2.5723 | Train Loss Components: Auxiliary: 0.7908 | Cardinality: 0.4742 | Classification: 0.6066 | Classification_binary: 0.5451 | HitGrouping: 0.3169 | Regression_cos_phi: 0.0050 | Regression_energy: 0.7066 | Regression_eta: 0.0113 | Regression_pt: 0.6924 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.3477 | Classification_binary: 0.5769 | Regression_cos_phi: 0.4157 | Regression_energy: 0.3653 | Regression_eta: 0.3266 | Regression_pt: 0.3712 | Regression_sin_phi: 0.3581 | LR: 1.00e-03 | DataLoad Time: 0.0169s | Model Forward Time: 4.4378s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.7377 | max_loss: 0.2512 | max_backward: 1.7506 | max_optimizer_step: 0.1548 | max_tensorboard_logging: 1.0398 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4776.7 | uss_MB: 3477.1 | child_rss_MB: 6372.1 | allocated_MB: 449.8 | reserved_MB: 24622.0 | inactive_split_MB: 902.2 +rank7 [2026-10-28 11:27:17,892] INFO : Step 300/50000 rank7 | Train Loss: 3.6172 | Train Loss Unweighted: 2.1934 | Train Loss Components: Auxiliary: 0.6840 | Cardinality: 0.3139 | Classification: 0.5005 | Classification_binary: 0.4575 | HitGrouping: 0.3097 | Regression_cos_phi: 0.0044 | Regression_energy: 0.6120 | Regression_eta: 0.0098 | Regression_pt: 0.6051 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2874 | Classification_binary: 0.4352 | Regression_cos_phi: 0.3684 | Regression_energy: 0.3508 | Regression_eta: 0.2806 | Regression_pt: 0.3590 | Regression_sin_phi: 0.2400 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.3679s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7416 | max_loss: 0.2092 | max_backward: 1.8436 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.9799 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4772.8 | uss_MB: 3482.4 | child_rss_MB: 6380.8 | allocated_MB: 446.5 | reserved_MB: 26624.0 | inactive_split_MB: 1012.0 +rank7 [2026-10-28 11:31:48,894] INFO : Step 400/50000 rank7 | Train Loss: 3.5512 | Train Loss Unweighted: 2.1747 | Train Loss Components: Auxiliary: 0.6912 | Cardinality: 0.3510 | Classification: 0.4506 | Classification_binary: 0.4446 | HitGrouping: 0.3081 | Regression_cos_phi: 0.0040 | Regression_energy: 0.6412 | Regression_eta: 0.0084 | Regression_pt: 0.6220 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2777 | Classification_binary: 0.4465 | Regression_cos_phi: 0.2362 | Regression_energy: 0.3374 | Regression_eta: 0.2180 | Regression_pt: 0.3385 | Regression_sin_phi: 0.2538 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.2439s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0470 | max_forward: 0.7363 | max_loss: 0.1458 | max_backward: 1.7200 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 0.9304 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0024 | Memory: rss_MB: 4771.8 | uss_MB: 3482.7 | child_rss_MB: 6403.6 | allocated_MB: 447.1 | reserved_MB: 26624.0 | inactive_split_MB: 1249.2 +rank7 [2026-10-28 11:36:20,165] INFO : Step 500/50000 rank7 | Train Loss: 3.7040 | Train Loss Unweighted: 2.1482 | Train Loss Components: Auxiliary: 0.6921 | Cardinality: 0.4270 | Classification: 0.4598 | Classification_binary: 0.5076 | HitGrouping: 0.2984 | Regression_cos_phi: 0.0040 | Regression_energy: 0.5766 | Regression_eta: 0.0093 | Regression_pt: 0.5865 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2645 | Classification_binary: 0.5370 | Regression_cos_phi: 0.3149 | Regression_energy: 0.2963 | Regression_eta: 0.2639 | Regression_pt: 0.3093 | Regression_sin_phi: 0.3469 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 4.4690s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7731 | max_loss: 0.2422 | max_backward: 1.9283 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9571 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.4 | uss_MB: 3482.0 | child_rss_MB: 6386.2 | allocated_MB: 449.6 | reserved_MB: 26624.0 | inactive_split_MB: 994.2 +rank7 [2026-10-28 11:40:54,778] INFO : Step 600/50000 rank7 | Train Loss: 3.3619 | Train Loss Unweighted: 2.0337 | Train Loss Components: Auxiliary: 0.6504 | Cardinality: 0.3441 | Classification: 0.4018 | Classification_binary: 0.4721 | HitGrouping: 0.2928 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5682 | Regression_eta: 0.0077 | Regression_pt: 0.5766 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2446 | Classification_binary: 0.4450 | Regression_cos_phi: 0.3400 | Regression_energy: 0.3312 | Regression_eta: 0.2600 | Regression_pt: 0.3478 | Regression_sin_phi: 0.2572 | LR: 1.00e-03 | DataLoad Time: 0.0143s | Model Forward Time: 4.5151s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0070 | max_forward: 0.8598 | max_loss: 0.1568 | max_backward: 1.8284 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4769.2 | uss_MB: 3486.5 | child_rss_MB: 6388.6 | allocated_MB: 450.5 | reserved_MB: 26624.0 | inactive_split_MB: 1047.1 +rank7 [2026-10-28 11:45:32,572] INFO : Step 700/50000 rank7 | Train Loss: 2.9731 | Train Loss Unweighted: 1.7530 | Train Loss Components: Auxiliary: 0.5807 | Cardinality: 0.3323 | Classification: 0.3542 | Classification_binary: 0.3997 | HitGrouping: 0.2932 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4915 | Regression_eta: 0.0061 | Regression_pt: 0.4953 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2135 | Classification_binary: 0.3845 | Regression_cos_phi: 0.2225 | Regression_energy: 0.2220 | Regression_eta: 0.1127 | Regression_pt: 0.2330 | Regression_sin_phi: 0.1537 | LR: 1.00e-03 | DataLoad Time: 0.0175s | Model Forward Time: 3.7402s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.6532 | max_loss: 0.1373 | max_backward: 1.5605 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.6012 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0056 | Memory: rss_MB: 4769.2 | uss_MB: 3483.5 | child_rss_MB: 6360.7 | allocated_MB: 447.5 | reserved_MB: 26626.0 | inactive_split_MB: 1239.2 +rank7 [2026-10-28 11:50:15,217] INFO : Step 800/50000 rank7 | Train Loss: 3.3787 | Train Loss Unweighted: 2.0524 | Train Loss Components: Auxiliary: 0.6518 | Cardinality: 0.3105 | Classification: 0.4343 | Classification_binary: 0.4498 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5749 | Regression_eta: 0.0083 | Regression_pt: 0.5776 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2566 | Classification_binary: 0.4555 | Regression_cos_phi: 0.2666 | Regression_energy: 0.3216 | Regression_eta: 0.2446 | Regression_pt: 0.3310 | Regression_sin_phi: 0.3313 | LR: 9.99e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2816s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8685 | max_loss: 0.2216 | max_backward: 2.3619 | max_optimizer_step: 0.1546 | max_tensorboard_logging: 1.2344 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0052 | Memory: rss_MB: 4767.8 | uss_MB: 3486.4 | child_rss_MB: 6386.5 | allocated_MB: 448.8 | reserved_MB: 28642.0 | inactive_split_MB: 1216.5 +rank7 [2026-10-28 11:54:58,535] INFO : Step 900/50000 rank7 | Train Loss: 3.3148 | Train Loss Unweighted: 1.9428 | Train Loss Components: Auxiliary: 0.6423 | Cardinality: 0.3627 | Classification: 0.3930 | Classification_binary: 0.4867 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0037 | Regression_energy: 0.5215 | Regression_eta: 0.0075 | Regression_pt: 0.5269 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2152 | Classification_binary: 0.4785 | Regression_cos_phi: 0.2554 | Regression_energy: 0.2580 | Regression_eta: 0.2339 | Regression_pt: 0.2721 | Regression_sin_phi: 0.2176 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4563s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.7729 | max_loss: 0.2515 | max_backward: 1.8521 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.9605 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4769.5 | uss_MB: 3487.7 | child_rss_MB: 6393.1 | allocated_MB: 451.1 | reserved_MB: 28642.0 | inactive_split_MB: 891.6 +rank7 [2026-10-28 11:59:55,247] INFO : Step 1000/50000 rank7 | Train Loss: 2.7920 | Train Loss Unweighted: 1.7295 | Train Loss Components: Auxiliary: 0.5723 | Cardinality: 0.2767 | Classification: 0.3190 | Classification_binary: 0.4086 | HitGrouping: 0.2904 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4963 | Regression_eta: 0.0053 | Regression_pt: 0.4951 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1821 | Classification_binary: 0.4111 | Regression_cos_phi: 0.1570 | Regression_energy: 0.2715 | Regression_eta: 0.1510 | Regression_pt: 0.2776 | Regression_sin_phi: 0.2089 | LR: 9.99e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.7549s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7924 | max_loss: 0.1664 | max_backward: 1.9505 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 1.2973 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 4768.1 | uss_MB: 3483.4 | child_rss_MB: 6372.3 | allocated_MB: 449.2 | reserved_MB: 28642.0 | inactive_split_MB: 1020.9 +rank7 [2026-10-28 11:59:56,332] INFO : Running validation on rank7 +rank7 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 11:59:56,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 12:00:48,695] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:00:49,300] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 12:01:16,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:01:17,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:01:18,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:01:19,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:01:29,890] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 12:01:30,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:01:30,839] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 12:01:46,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:01:47,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:01:48,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:01:49,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:01:59,754] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:02:00,466] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:02:01,007] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 12:02:16,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:02:17,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:02:18,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:02:19,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_1000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:02:29,125] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 12:07:33,552] INFO : Step 1100/50000 rank7 | Train Loss: 2.9727 | Train Loss Unweighted: 1.8050 | Train Loss Components: Auxiliary: 0.5969 | Cardinality: 0.2977 | Classification: 0.3569 | Classification_binary: 0.4130 | HitGrouping: 0.2862 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5091 | Regression_eta: 0.0062 | Regression_pt: 0.5138 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2314 | Classification_binary: 0.4066 | Regression_cos_phi: 0.2956 | Regression_energy: 0.2734 | Regression_eta: 0.2078 | Regression_pt: 0.2787 | Regression_sin_phi: 0.1696 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8134s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0982 | max_forward: 0.8238 | max_loss: 0.1457 | max_backward: 1.9911 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 1.0251 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6504.7 | uss_MB: 5215.0 | child_rss_MB: 6394.9 | allocated_MB: 448.3 | reserved_MB: 28340.0 | inactive_split_MB: 1399.3 +rank7 [2026-10-28 12:12:20,261] INFO : Step 1200/50000 rank7 | Train Loss: 3.2531 | Train Loss Unweighted: 1.8415 | Train Loss Components: Auxiliary: 0.6362 | Cardinality: 0.4015 | Classification: 0.3682 | Classification_binary: 0.4739 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0035 | Regression_energy: 0.4895 | Regression_eta: 0.0067 | Regression_pt: 0.4961 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2088 | Classification_binary: 0.4814 | Regression_cos_phi: 0.2085 | Regression_energy: 0.2398 | Regression_eta: 0.1686 | Regression_pt: 0.2505 | Regression_sin_phi: 0.2260 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4365s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0075 | max_forward: 0.7790 | max_loss: 0.4823 | max_backward: 1.9216 | max_optimizer_step: 0.2780 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6500.5 | uss_MB: 5212.6 | child_rss_MB: 6419.3 | allocated_MB: 452.6 | reserved_MB: 28342.0 | inactive_split_MB: 987.4 +rank7 [2026-10-28 12:17:07,816] INFO : Step 1300/50000 rank7 | Train Loss: 2.8295 | Train Loss Unweighted: 1.6506 | Train Loss Components: Auxiliary: 0.5563 | Cardinality: 0.2932 | Classification: 0.3276 | Classification_binary: 0.4047 | HitGrouping: 0.2736 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4491 | Regression_eta: 0.0063 | Regression_pt: 0.4569 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2420 | Classification_binary: 0.4390 | Regression_cos_phi: 0.2383 | Regression_energy: 0.2421 | Regression_eta: 0.1906 | Regression_pt: 0.2549 | Regression_sin_phi: 0.2203 | LR: 9.98e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2268s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1025 | max_forward: 0.8397 | max_loss: 0.1473 | max_backward: 1.8848 | max_optimizer_step: 0.1691 | max_tensorboard_logging: 0.5291 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0130 | Memory: rss_MB: 5630.2 | uss_MB: 4343.8 | child_rss_MB: 6409.3 | allocated_MB: 450.1 | reserved_MB: 28342.0 | inactive_split_MB: 983.3 +rank7 [2026-10-28 12:21:55,309] INFO : Step 1400/50000 rank7 | Train Loss: 2.8998 | Train Loss Unweighted: 1.6575 | Train Loss Components: Auxiliary: 0.5551 | Cardinality: 0.3004 | Classification: 0.3372 | Classification_binary: 0.3796 | HitGrouping: 0.2826 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4626 | Regression_eta: 0.0068 | Regression_pt: 0.4646 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1933 | Classification_binary: 0.3832 | Regression_cos_phi: 0.3078 | Regression_energy: 0.2150 | Regression_eta: 0.1751 | Regression_pt: 0.2258 | Regression_sin_phi: 0.1780 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5550s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7345 | max_loss: 0.1444 | max_backward: 1.7078 | max_optimizer_step: 0.2412 | max_tensorboard_logging: 1.0733 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0489 | Memory: rss_MB: 5625.4 | uss_MB: 4343.9 | child_rss_MB: 6400.7 | allocated_MB: 449.1 | reserved_MB: 28342.0 | inactive_split_MB: 1366.0 +rank7 [2026-10-28 12:26:33,582] INFO : Step 1500/50000 rank7 | Train Loss: 2.8869 | Train Loss Unweighted: 1.7047 | Train Loss Components: Auxiliary: 0.5699 | Cardinality: 0.2860 | Classification: 0.3479 | Classification_binary: 0.4316 | HitGrouping: 0.2823 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4527 | Regression_eta: 0.0063 | Regression_pt: 0.4602 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2025 | Classification_binary: 0.4286 | Regression_cos_phi: 0.2116 | Regression_energy: 0.2494 | Regression_eta: 0.1780 | Regression_pt: 0.2587 | Regression_sin_phi: 0.2615 | LR: 9.98e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9971s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.8359 | max_loss: 0.1566 | max_backward: 2.1157 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 1.1803 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5607.4 | uss_MB: 4325.3 | child_rss_MB: 6411.7 | allocated_MB: 450.0 | reserved_MB: 28342.0 | inactive_split_MB: 1167.0 +rank7 [2026-10-28 12:31:13,937] INFO : Step 1600/50000 rank7 | Train Loss: 2.8138 | Train Loss Unweighted: 1.6281 | Train Loss Components: Auxiliary: 0.5433 | Cardinality: 0.2713 | Classification: 0.3076 | Classification_binary: 0.4297 | HitGrouping: 0.2739 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4363 | Regression_eta: 0.0064 | Regression_pt: 0.4419 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1760 | Classification_binary: 0.4114 | Regression_cos_phi: 0.2205 | Regression_energy: 0.2288 | Regression_eta: 0.1673 | Regression_pt: 0.2373 | Regression_sin_phi: 0.2155 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8058s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0066 | max_forward: 0.8023 | max_loss: 0.2527 | max_backward: 1.9566 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5619.2 | uss_MB: 4345.4 | child_rss_MB: 6402.1 | allocated_MB: 447.8 | reserved_MB: 28342.0 | inactive_split_MB: 1396.1 +rank7 [2026-10-28 12:36:04,097] INFO : Step 1700/50000 rank7 | Train Loss: 2.4955 | Train Loss Unweighted: 1.4832 | Train Loss Components: Auxiliary: 0.4929 | Cardinality: 0.2330 | Classification: 0.2730 | Classification_binary: 0.3433 | HitGrouping: 0.2694 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4283 | Regression_eta: 0.0056 | Regression_pt: 0.4278 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1732 | Classification_binary: 0.3254 | Regression_cos_phi: 0.1893 | Regression_energy: 0.2105 | Regression_eta: 0.1411 | Regression_pt: 0.2154 | Regression_sin_phi: 0.1626 | LR: 9.97e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6759s | Phase Times: max_pre_step_barrier: 0.1117 | max_batch_to_device: 0.0065 | max_forward: 0.7232 | max_loss: 0.1407 | max_backward: 1.7761 | max_optimizer_step: 0.1723 | max_tensorboard_logging: 1.1073 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 5618.8 | uss_MB: 4345.1 | child_rss_MB: 6391.4 | allocated_MB: 444.7 | reserved_MB: 28342.0 | inactive_split_MB: 1030.7 +rank7 [2026-10-28 12:40:54,250] INFO : Step 1800/50000 rank7 | Train Loss: 2.7104 | Train Loss Unweighted: 1.6188 | Train Loss Components: Auxiliary: 0.5398 | Cardinality: 0.2781 | Classification: 0.3223 | Classification_binary: 0.3911 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4446 | Regression_eta: 0.0056 | Regression_pt: 0.4499 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.3991 | Regression_cos_phi: 0.1808 | Regression_energy: 0.2445 | Regression_eta: 0.1437 | Regression_pt: 0.2542 | Regression_sin_phi: 0.1787 | LR: 9.97e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6768s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8046 | max_loss: 0.1465 | max_backward: 1.9913 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 1.0097 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0097 | Memory: rss_MB: 5621.9 | uss_MB: 4345.1 | child_rss_MB: 6415.9 | allocated_MB: 448.5 | reserved_MB: 28342.0 | inactive_split_MB: 1008.0 +rank7 [2026-10-28 12:45:37,436] INFO : Step 1900/50000 rank7 | Train Loss: 2.8696 | Train Loss Unweighted: 1.6551 | Train Loss Components: Auxiliary: 0.5521 | Cardinality: 0.3078 | Classification: 0.3237 | Classification_binary: 0.4465 | HitGrouping: 0.2669 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4323 | Regression_eta: 0.0064 | Regression_pt: 0.4400 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2031 | Classification_binary: 0.4733 | Regression_cos_phi: 0.2678 | Regression_energy: 0.2213 | Regression_eta: 0.2130 | Regression_pt: 0.2316 | Regression_sin_phi: 0.2536 | LR: 9.97e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4275s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0067 | max_forward: 0.8255 | max_loss: 0.2389 | max_backward: 1.8762 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.6219 | max_tensorboard_flush: 0.0156 | max_post_step_barrier: 0.0552 | Memory: rss_MB: 5612.2 | uss_MB: 4345.0 | child_rss_MB: 6407.1 | allocated_MB: 450.1 | reserved_MB: 30550.0 | inactive_split_MB: 988.1 +rank7 [2026-10-28 12:50:22,600] INFO : Step 2000/50000 rank7 | Train Loss: 2.4875 | Train Loss Unweighted: 1.4740 | Train Loss Components: Auxiliary: 0.4866 | Cardinality: 0.2180 | Classification: 0.2573 | Classification_binary: 0.3684 | HitGrouping: 0.2682 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4192 | Regression_eta: 0.0056 | Regression_pt: 0.4182 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3996 | Regression_cos_phi: 0.1865 | Regression_energy: 0.2239 | Regression_eta: 0.1708 | Regression_pt: 0.2247 | Regression_sin_phi: 0.2096 | LR: 9.96e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9885s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7961 | max_loss: 0.4975 | max_backward: 1.9912 | max_optimizer_step: 0.2832 | max_tensorboard_logging: 1.0058 | max_tensorboard_flush: 0.0033 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 5609.5 | uss_MB: 4006.3 | child_rss_MB: 6431.7 | allocated_MB: 445.9 | reserved_MB: 30550.0 | inactive_split_MB: 1396.1 +rank7 [2026-10-28 12:50:23,688] INFO : Running validation on rank7 +rank7 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 12:50:23,689] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 12:51:19,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:51:20,336] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 12:51:47,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:51:48,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:51:49,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:51:50,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:52:00,925] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 12:52:01,713] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:52:02,240] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 12:52:18,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:52:19,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:52:20,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:52:21,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:52:31,021] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:52:31,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:52:32,091] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 12:52:48,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:52:48,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:52:49,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:52:50,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_2000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:53:01,005] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 12:58:21,945] INFO : Step 2100/50000 rank7 | Train Loss: 2.4839 | Train Loss Unweighted: 1.5025 | Train Loss Components: Auxiliary: 0.4936 | Cardinality: 0.2217 | Classification: 0.2888 | Classification_binary: 0.3469 | HitGrouping: 0.2676 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4264 | Regression_eta: 0.0050 | Regression_pt: 0.4301 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2010 | Classification_binary: 0.3951 | Regression_cos_phi: 0.1999 | Regression_energy: 0.2234 | Regression_eta: 0.1466 | Regression_pt: 0.2331 | Regression_sin_phi: 0.1829 | LR: 9.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6449s | Phase Times: max_pre_step_barrier: 0.0120 | max_batch_to_device: 0.0068 | max_forward: 0.7390 | max_loss: 0.1487 | max_backward: 1.8405 | max_optimizer_step: 0.2868 | max_tensorboard_logging: 0.9207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6543.2 | uss_MB: 5275.8 | child_rss_MB: 6440.7 | allocated_MB: 451.1 | reserved_MB: 24486.0 | inactive_split_MB: 1197.3 +rank7 [2026-10-28 13:03:25,047] INFO : Step 2200/50000 rank7 | Train Loss: 2.8311 | Train Loss Unweighted: 1.6845 | Train Loss Components: Auxiliary: 0.5516 | Cardinality: 0.2685 | Classification: 0.3231 | Classification_binary: 0.4170 | HitGrouping: 0.2689 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4624 | Regression_eta: 0.0063 | Regression_pt: 0.4694 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.2044 | Classification_binary: 0.4315 | Regression_cos_phi: 0.2735 | Regression_energy: 0.2426 | Regression_eta: 0.1730 | Regression_pt: 0.2492 | Regression_sin_phi: 0.2273 | LR: 9.96e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.4355s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.7748 | max_loss: 0.1498 | max_backward: 3.5600 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 1.0115 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6531.5 | uss_MB: 5275.4 | child_rss_MB: 6441.0 | allocated_MB: 448.5 | reserved_MB: 25020.0 | inactive_split_MB: 1461.7 +rank7 [2026-10-28 13:08:12,324] INFO : Step 2300/50000 rank7 | Train Loss: 2.8327 | Train Loss Unweighted: 1.5894 | Train Loss Components: Auxiliary: 0.5364 | Cardinality: 0.3660 | Classification: 0.2920 | Classification_binary: 0.4007 | HitGrouping: 0.2692 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4386 | Regression_eta: 0.0067 | Regression_pt: 0.4457 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1460 | Classification_binary: 0.3741 | Regression_cos_phi: 0.1962 | Regression_energy: 0.2127 | Regression_eta: 0.2037 | Regression_pt: 0.2189 | Regression_sin_phi: 0.1966 | LR: 9.95e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.1086s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0064 | max_forward: 1.2342 | max_loss: 0.4937 | max_backward: 2.1725 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.9137 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5697.5 | uss_MB: 4443.4 | child_rss_MB: 6451.7 | allocated_MB: 447.5 | reserved_MB: 26402.0 | inactive_split_MB: 1029.5 +rank7 [2026-10-28 13:13:00,653] INFO : Step 2400/50000 rank7 | Train Loss: 2.4857 | Train Loss Unweighted: 1.4676 | Train Loss Components: Auxiliary: 0.4905 | Cardinality: 0.2335 | Classification: 0.2601 | Classification_binary: 0.3231 | HitGrouping: 0.2558 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4369 | Regression_eta: 0.0053 | Regression_pt: 0.4367 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1723 | Classification_binary: 0.3224 | Regression_cos_phi: 0.2560 | Regression_energy: 0.2253 | Regression_eta: 0.1639 | Regression_pt: 0.2283 | Regression_sin_phi: 0.2235 | LR: 9.95e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2182s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7450 | max_loss: 0.1453 | max_backward: 1.9102 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.5890 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5698.4 | uss_MB: 4444.1 | child_rss_MB: 6455.9 | allocated_MB: 444.7 | reserved_MB: 26830.0 | inactive_split_MB: 1232.7 +rank7 [2026-10-28 13:18:06,915] INFO : Step 2500/50000 rank7 | Train Loss: 2.7497 | Train Loss Unweighted: 1.5725 | Train Loss Components: Auxiliary: 0.5248 | Cardinality: 0.2678 | Classification: 0.3257 | Classification_binary: 0.4066 | HitGrouping: 0.2573 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4117 | Regression_eta: 0.0064 | Regression_pt: 0.4157 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1807 | Classification_binary: 0.3922 | Regression_cos_phi: 0.2504 | Regression_energy: 0.2116 | Regression_eta: 0.1762 | Regression_pt: 0.2162 | Regression_sin_phi: 0.2024 | LR: 9.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.0314s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.8164 | max_loss: 0.1524 | max_backward: 2.0956 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 0.6549 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 5687.3 | uss_MB: 4405.0 | child_rss_MB: 6449.5 | allocated_MB: 450.0 | reserved_MB: 26830.0 | inactive_split_MB: 1235.9 +rank7 [2026-10-28 13:22:57,736] INFO : Step 2600/50000 rank7 | Train Loss: 2.7719 | Train Loss Unweighted: 1.6485 | Train Loss Components: Auxiliary: 0.5438 | Cardinality: 0.2914 | Classification: 0.2867 | Classification_binary: 0.4203 | HitGrouping: 0.2619 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4622 | Regression_eta: 0.0056 | Regression_pt: 0.4679 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1618 | Classification_binary: 0.3937 | Regression_cos_phi: 0.1979 | Regression_energy: 0.2542 | Regression_eta: 0.1578 | Regression_pt: 0.2640 | Regression_sin_phi: 0.1848 | LR: 9.94e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3782s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.8224 | max_loss: 0.2337 | max_backward: 1.8825 | max_optimizer_step: 0.2673 | max_tensorboard_logging: 0.5755 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0183 | Memory: rss_MB: 5695.9 | uss_MB: 4443.9 | child_rss_MB: 6455.1 | allocated_MB: 453.4 | reserved_MB: 26832.0 | inactive_split_MB: 1373.2 +rank7 [2026-10-28 13:27:39,836] INFO : Step 2700/50000 rank7 | Train Loss: 2.3898 | Train Loss Unweighted: 1.3408 | Train Loss Components: Auxiliary: 0.4591 | Cardinality: 0.2891 | Classification: 0.2369 | Classification_binary: 0.3241 | HitGrouping: 0.2569 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3848 | Regression_eta: 0.0050 | Regression_pt: 0.3850 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1396 | Classification_binary: 0.3103 | Regression_cos_phi: 0.1743 | Regression_energy: 0.2001 | Regression_eta: 0.1236 | Regression_pt: 0.2061 | Regression_sin_phi: 0.1411 | LR: 9.94e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5211s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0468 | max_forward: 0.7435 | max_loss: 0.1492 | max_backward: 1.8101 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.5969 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5695.8 | uss_MB: 4443.3 | child_rss_MB: 6462.2 | allocated_MB: 448.6 | reserved_MB: 28434.0 | inactive_split_MB: 1140.5 +rank7 [2026-10-28 13:32:27,614] INFO : Step 2800/50000 rank7 | Train Loss: 2.5037 | Train Loss Unweighted: 1.4785 | Train Loss Components: Auxiliary: 0.4880 | Cardinality: 0.2418 | Classification: 0.2774 | Classification_binary: 0.3472 | HitGrouping: 0.2602 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4197 | Regression_eta: 0.0051 | Regression_pt: 0.4236 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1842 | Classification_binary: 0.3976 | Regression_cos_phi: 0.2563 | Regression_energy: 0.1994 | Regression_eta: 0.1896 | Regression_pt: 0.2109 | Regression_sin_phi: 0.2167 | LR: 9.93e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2920s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1043 | max_forward: 0.8124 | max_loss: 0.1470 | max_backward: 1.9740 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5696.4 | uss_MB: 4443.5 | child_rss_MB: 6462.9 | allocated_MB: 448.7 | reserved_MB: 30438.0 | inactive_split_MB: 1466.8 +rank7 [2026-10-28 13:37:09,431] INFO : Step 2900/50000 rank7 | Train Loss: 2.8268 | Train Loss Unweighted: 1.6136 | Train Loss Components: Auxiliary: 0.5310 | Cardinality: 0.2859 | Classification: 0.2990 | Classification_binary: 0.4249 | HitGrouping: 0.2616 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4348 | Regression_eta: 0.0067 | Regression_pt: 0.4415 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.4284 | Regression_cos_phi: 0.2808 | Regression_energy: 0.2300 | Regression_eta: 0.1962 | Regression_pt: 0.2404 | Regression_sin_phi: 0.2333 | LR: 9.93e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5097s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7666 | max_loss: 0.1663 | max_backward: 1.7209 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8320 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5698.4 | uss_MB: 4443.2 | child_rss_MB: 6458.2 | allocated_MB: 450.6 | reserved_MB: 30438.0 | inactive_split_MB: 1043.4 +rank7 [2026-10-28 13:42:03,678] INFO : Step 3000/50000 rank7 | Train Loss: 2.4249 | Train Loss Unweighted: 1.3719 | Train Loss Components: Auxiliary: 0.4577 | Cardinality: 0.2547 | Classification: 0.2481 | Classification_binary: 0.3586 | HitGrouping: 0.2498 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3754 | Regression_eta: 0.0051 | Regression_pt: 0.3792 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1451 | Classification_binary: 0.3527 | Regression_cos_phi: 0.2233 | Regression_energy: 0.1881 | Regression_eta: 0.1471 | Regression_pt: 0.1936 | Regression_sin_phi: 0.1938 | LR: 9.92e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5876s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7395 | max_loss: 0.2544 | max_backward: 1.7820 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.9841 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 5691.1 | uss_MB: 4442.0 | child_rss_MB: 6463.6 | allocated_MB: 444.3 | reserved_MB: 30438.0 | inactive_split_MB: 1467.8 +rank7 [2026-10-28 13:42:04,678] INFO : Running validation on rank7 +rank7 [2026-10-28 13:42:04,679] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:42:04,679] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:42:57,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:42:58,469] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 13:43:25,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:43:26,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:43:27,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:43:28,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:43:38,226] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:43:38,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:43:39,344] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 13:43:55,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:43:56,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:43:57,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:43:58,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:44:08,223] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:44:08,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:44:09,101] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 13:44:24,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:44:25,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:44:26,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:44:27,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_3000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:44:37,180] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 13:49:29,793] INFO : Step 3100/50000 rank7 | Train Loss: 2.2982 | Train Loss Unweighted: 1.3383 | Train Loss Components: Auxiliary: 0.4426 | Cardinality: 0.2252 | Classification: 0.2519 | Classification_binary: 0.3088 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3820 | Regression_eta: 0.0047 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1610 | Regression_energy: 0.2144 | Regression_eta: 0.1470 | Regression_pt: 0.2224 | Regression_sin_phi: 0.1672 | LR: 9.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8442s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7683 | max_loss: 0.1446 | max_backward: 2.0155 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 1.0077 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6832.6 | uss_MB: 5582.9 | child_rss_MB: 6431.2 | allocated_MB: 449.2 | reserved_MB: 26186.0 | inactive_split_MB: 1386.9 +rank7 [2026-10-28 13:54:22,656] INFO : Step 3200/50000 rank7 | Train Loss: 2.6332 | Train Loss Unweighted: 1.5005 | Train Loss Components: Auxiliary: 0.4902 | Cardinality: 0.2402 | Classification: 0.2933 | Classification_binary: 0.4028 | HitGrouping: 0.2566 | Regression_cos_phi: 0.0033 | Regression_energy: 0.3933 | Regression_eta: 0.0057 | Regression_pt: 0.3988 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1848 | Classification_binary: 0.4256 | Regression_cos_phi: 0.2621 | Regression_energy: 0.1987 | Regression_eta: 0.1572 | Regression_pt: 0.2080 | Regression_sin_phi: 0.2500 | LR: 9.91e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.0540s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0072 | max_forward: 0.9463 | max_loss: 0.2307 | max_backward: 1.9710 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9890 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6833.8 | uss_MB: 5582.9 | child_rss_MB: 6428.3 | allocated_MB: 451.1 | reserved_MB: 28226.0 | inactive_split_MB: 1176.4 +rank7 [2026-10-28 13:58:57,113] INFO : Step 3300/50000 rank7 | Train Loss: 2.4382 | Train Loss Unweighted: 1.4232 | Train Loss Components: Auxiliary: 0.4675 | Cardinality: 0.2204 | Classification: 0.2733 | Classification_binary: 0.3681 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3814 | Regression_eta: 0.0055 | Regression_pt: 0.3894 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1796 | Classification_binary: 0.3861 | Regression_cos_phi: 0.2562 | Regression_energy: 0.2141 | Regression_eta: 0.1892 | Regression_pt: 0.2265 | Regression_sin_phi: 0.1984 | LR: 9.90e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7399s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0703 | max_forward: 0.9374 | max_loss: 0.1485 | max_backward: 1.9768 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.7475 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5799.4 | uss_MB: 4552.6 | child_rss_MB: 6441.3 | allocated_MB: 450.5 | reserved_MB: 28228.0 | inactive_split_MB: 981.5 +rank7 [2026-10-28 14:03:33,298] INFO : Step 3400/50000 rank7 | Train Loss: 2.2888 | Train Loss Unweighted: 1.3477 | Train Loss Components: Auxiliary: 0.4500 | Cardinality: 0.2378 | Classification: 0.2273 | Classification_binary: 0.2975 | HitGrouping: 0.2502 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4065 | Regression_eta: 0.0051 | Regression_pt: 0.4067 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1210 | Classification_binary: 0.2958 | Regression_cos_phi: 0.1619 | Regression_energy: 0.2170 | Regression_eta: 0.1233 | Regression_pt: 0.2254 | Regression_sin_phi: 0.2054 | LR: 9.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9442s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7652 | max_loss: 0.1453 | max_backward: 1.5461 | max_optimizer_step: 0.1915 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1202 | Memory: rss_MB: 5807.1 | uss_MB: 4563.6 | child_rss_MB: 6427.9 | allocated_MB: 446.7 | reserved_MB: 28232.0 | inactive_split_MB: 1076.0 +rank7 [2026-10-28 14:08:16,394] INFO : Step 3500/50000 rank7 | Train Loss: 2.4030 | Train Loss Unweighted: 1.4035 | Train Loss Components: Auxiliary: 0.4639 | Cardinality: 0.2404 | Classification: 0.2697 | Classification_binary: 0.3577 | HitGrouping: 0.2482 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3801 | Regression_eta: 0.0049 | Regression_pt: 0.3859 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1622 | Classification_binary: 0.3468 | Regression_cos_phi: 0.2075 | Regression_energy: 0.2317 | Regression_eta: 0.1343 | Regression_pt: 0.2345 | Regression_sin_phi: 0.1750 | LR: 9.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7444s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8310 | max_loss: 0.1679 | max_backward: 2.2472 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.7198 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0242 | Memory: rss_MB: 5776.5 | uss_MB: 4554.6 | child_rss_MB: 6335.9 | allocated_MB: 451.7 | reserved_MB: 28232.0 | inactive_split_MB: 1185.2 +rank7 [2026-10-28 14:13:02,150] INFO : Step 3600/50000 rank7 | Train Loss: 2.5473 | Train Loss Unweighted: 1.4700 | Train Loss Components: Auxiliary: 0.4732 | Cardinality: 0.2555 | Classification: 0.2601 | Classification_binary: 0.4099 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3919 | Regression_eta: 0.0052 | Regression_pt: 0.3970 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1685 | Classification_binary: 0.3908 | Regression_cos_phi: 0.2208 | Regression_energy: 0.2118 | Regression_eta: 0.1476 | Regression_pt: 0.2210 | Regression_sin_phi: 0.1960 | LR: 9.89e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8773s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8182 | max_loss: 0.1573 | max_backward: 1.9537 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8374 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 5785.8 | uss_MB: 4564.2 | child_rss_MB: 6350.3 | allocated_MB: 449.7 | reserved_MB: 28232.0 | inactive_split_MB: 940.8 +rank7 [2026-10-28 14:17:42,402] INFO : Step 3700/50000 rank7 | Train Loss: 2.1842 | Train Loss Unweighted: 1.2555 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.1924 | Classification: 0.2286 | Classification_binary: 0.3206 | HitGrouping: 0.2491 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3463 | Regression_eta: 0.0051 | Regression_pt: 0.3500 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1286 | Classification_binary: 0.3472 | Regression_cos_phi: 0.2142 | Regression_energy: 0.1866 | Regression_eta: 0.1480 | Regression_pt: 0.1977 | Regression_sin_phi: 0.1929 | LR: 9.88e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.2534s | Phase Times: max_pre_step_barrier: 0.0219 | max_batch_to_device: 0.0752 | max_forward: 1.0106 | max_loss: 0.2756 | max_backward: 2.5517 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.5146 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 5784.5 | uss_MB: 4218.3 | child_rss_MB: 6354.6 | allocated_MB: 447.7 | reserved_MB: 28232.0 | inactive_split_MB: 1389.4 +rank7 [2026-10-28 14:22:44,342] INFO : Step 3800/50000 rank7 | Train Loss: 2.3766 | Train Loss Unweighted: 1.3767 | Train Loss Components: Auxiliary: 0.4529 | Cardinality: 0.2400 | Classification: 0.2427 | Classification_binary: 0.3290 | HitGrouping: 0.2506 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3934 | Regression_eta: 0.0053 | Regression_pt: 0.4011 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1652 | Classification_binary: 0.3427 | Regression_cos_phi: 0.1699 | Regression_energy: 0.2163 | Regression_eta: 0.1596 | Regression_pt: 0.2285 | Regression_sin_phi: 0.2358 | LR: 9.87e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1806s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7811 | max_loss: 0.1495 | max_backward: 1.7224 | max_optimizer_step: 0.2450 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0249 | Memory: rss_MB: 5766.9 | uss_MB: 4562.6 | child_rss_MB: 6360.7 | allocated_MB: 447.6 | reserved_MB: 28232.0 | inactive_split_MB: 1178.4 +rank7 [2026-10-28 14:27:33,636] INFO : Step 3900/50000 rank7 | Train Loss: 2.7008 | Train Loss Unweighted: 1.4619 | Train Loss Components: Auxiliary: 0.5080 | Cardinality: 0.3987 | Classification: 0.2667 | Classification_binary: 0.3944 | HitGrouping: 0.2477 | Regression_cos_phi: 0.0030 | Regression_energy: 0.3927 | Regression_eta: 0.0050 | Regression_pt: 0.3973 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1617 | Classification_binary: 0.4159 | Regression_cos_phi: 0.2274 | Regression_energy: 0.1922 | Regression_eta: 0.1385 | Regression_pt: 0.2020 | Regression_sin_phi: 0.2273 | LR: 9.87e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.8199s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0085 | max_forward: 0.8287 | max_loss: 0.1529 | max_backward: 2.5623 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5767.1 | uss_MB: 4435.9 | child_rss_MB: 6341.2 | allocated_MB: 450.0 | reserved_MB: 28232.0 | inactive_split_MB: 992.0 +rank7 [2026-10-28 14:32:38,645] INFO : Step 4000/50000 rank7 | Train Loss: 2.2797 | Train Loss Unweighted: 1.3353 | Train Loss Components: Auxiliary: 0.4281 | Cardinality: 0.2169 | Classification: 0.2160 | Classification_binary: 0.3428 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3812 | Regression_eta: 0.0046 | Regression_pt: 0.3854 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1181 | Classification_binary: 0.2897 | Regression_cos_phi: 0.2105 | Regression_energy: 0.2037 | Regression_eta: 0.1223 | Regression_pt: 0.2120 | Regression_sin_phi: 0.1599 | LR: 9.86e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5854s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0487 | max_loss: 0.1458 | max_backward: 1.8288 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5763.8 | uss_MB: 4564.8 | child_rss_MB: 6358.9 | allocated_MB: 446.3 | reserved_MB: 28232.0 | inactive_split_MB: 925.3 +rank7 [2026-10-28 14:32:39,608] INFO : Running validation on rank7 +rank7 [2026-10-28 14:32:39,608] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 14:32:39,609] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 14:33:34,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:33:34,720] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 14:34:03,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 14:34:04,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 14:34:08,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 14:34:11,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 14:34:21,118] INFO : Time taken to make predictions on device 7 is: 0.34 min +rank7 [2026-10-28 14:34:23,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:34:24,281] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 14:34:40,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 14:34:41,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 14:34:43,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 14:34:44,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 14:34:54,841] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-28 14:34:55,796] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:34:56,345] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 14:35:11,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 14:35:12,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 14:35:13,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 14:35:13,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_4000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 14:35:23,988] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 14:40:25,279] INFO : Step 4100/50000 rank7 | Train Loss: 2.2247 | Train Loss Unweighted: 1.3117 | Train Loss Components: Auxiliary: 0.4222 | Cardinality: 0.1952 | Classification: 0.2176 | Classification_binary: 0.3103 | HitGrouping: 0.2466 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3871 | Regression_eta: 0.0048 | Regression_pt: 0.3869 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 0.3281 | Regression_cos_phi: 0.2099 | Regression_energy: 0.1882 | Regression_eta: 0.1211 | Regression_pt: 0.1948 | Regression_sin_phi: 0.1467 | LR: 9.85e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8513s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.7561 | max_loss: 0.1473 | max_backward: 1.8347 | max_optimizer_step: 0.2595 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0940 | Memory: rss_MB: 6754.7 | uss_MB: 5553.0 | child_rss_MB: 6391.3 | allocated_MB: 446.7 | reserved_MB: 25356.0 | inactive_split_MB: 980.4 +rank7 [2026-10-28 14:45:08,006] INFO : Step 4200/50000 rank7 | Train Loss: 2.4393 | Train Loss Unweighted: 1.4368 | Train Loss Components: Auxiliary: 0.4612 | Cardinality: 0.2221 | Classification: 0.2879 | Classification_binary: 0.3637 | HitGrouping: 0.2443 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3855 | Regression_eta: 0.0055 | Regression_pt: 0.3887 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1590 | Classification_binary: 0.3975 | Regression_cos_phi: 0.2030 | Regression_energy: 0.1761 | Regression_eta: 0.1415 | Regression_pt: 0.1819 | Regression_sin_phi: 0.2050 | LR: 9.84e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5084s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0065 | max_forward: 0.8425 | max_loss: 0.1617 | max_backward: 1.9171 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5829 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6756.2 | uss_MB: 5554.3 | child_rss_MB: 6385.9 | allocated_MB: 451.6 | reserved_MB: 27358.0 | inactive_split_MB: 1422.9 +rank7 [2026-10-28 14:49:55,511] INFO : Step 4300/50000 rank7 | Train Loss: 2.3580 | Train Loss Unweighted: 1.3692 | Train Loss Components: Auxiliary: 0.4401 | Cardinality: 0.2284 | Classification: 0.2569 | Classification_binary: 0.3680 | HitGrouping: 0.2407 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3652 | Regression_eta: 0.0051 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1441 | Classification_binary: 0.3630 | Regression_cos_phi: 0.1882 | Regression_energy: 0.1995 | Regression_eta: 0.1322 | Regression_pt: 0.2037 | Regression_sin_phi: 0.1821 | LR: 9.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4596s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7893 | max_loss: 0.1545 | max_backward: 1.6763 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.7439 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5839.7 | uss_MB: 4640.1 | child_rss_MB: 6385.4 | allocated_MB: 446.6 | reserved_MB: 27358.0 | inactive_split_MB: 968.3 +rank7 [2026-10-28 14:54:31,942] INFO : Step 4400/50000 rank7 | Train Loss: 2.0697 | Train Loss Unweighted: 1.2145 | Train Loss Components: Auxiliary: 0.3987 | Cardinality: 0.2086 | Classification: 0.1987 | Classification_binary: 0.2861 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3593 | Regression_eta: 0.0042 | Regression_pt: 0.3620 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 0.2734 | Regression_cos_phi: 0.1599 | Regression_energy: 0.1528 | Regression_eta: 0.1585 | Regression_pt: 0.1557 | Regression_sin_phi: 0.1753 | LR: 9.83e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1737s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7711 | max_loss: 0.1468 | max_backward: 1.7687 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.5885 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5848.6 | uss_MB: 4650.4 | child_rss_MB: 6379.0 | allocated_MB: 445.6 | reserved_MB: 27362.0 | inactive_split_MB: 1071.5 +rank7 [2026-10-28 14:59:28,686] INFO : Step 4500/50000 rank7 | Train Loss: 2.3351 | Train Loss Unweighted: 1.3242 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2494 | Classification: 0.2488 | Classification_binary: 0.3388 | HitGrouping: 0.2446 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3607 | Regression_eta: 0.0048 | Regression_pt: 0.3657 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1504 | Classification_binary: 0.3492 | Regression_cos_phi: 0.1830 | Regression_energy: 0.1870 | Regression_eta: 0.1319 | Regression_pt: 0.1959 | Regression_sin_phi: 0.1749 | LR: 9.82e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2508s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.9802 | max_loss: 0.2489 | max_backward: 1.8440 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4709 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1282 | Memory: rss_MB: 5849.5 | uss_MB: 4651.3 | child_rss_MB: 6381.1 | allocated_MB: 449.0 | reserved_MB: 27366.0 | inactive_split_MB: 1358.9 +rank7 [2026-10-28 15:04:24,811] INFO : Step 4600/50000 rank7 | Train Loss: 2.4892 | Train Loss Unweighted: 1.3960 | Train Loss Components: Auxiliary: 0.4677 | Cardinality: 0.2934 | Classification: 0.2445 | Classification_binary: 0.4159 | HitGrouping: 0.2429 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3600 | Regression_eta: 0.0051 | Regression_pt: 0.3653 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.4071 | Regression_cos_phi: 0.2526 | Regression_energy: 0.2012 | Regression_eta: 0.1494 | Regression_pt: 0.2085 | Regression_sin_phi: 0.2094 | LR: 9.81e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.7030s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8339 | max_loss: 0.2572 | max_backward: 2.2051 | max_optimizer_step: 0.1935 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0006 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5849.8 | uss_MB: 4653.3 | child_rss_MB: 6382.2 | allocated_MB: 450.1 | reserved_MB: 29402.0 | inactive_split_MB: 1357.9 +rank7 [2026-10-28 15:09:18,560] INFO : Step 4700/50000 rank7 | Train Loss: 2.0569 | Train Loss Unweighted: 1.1959 | Train Loss Components: Auxiliary: 0.3892 | Cardinality: 0.2086 | Classification: 0.1901 | Classification_binary: 0.3112 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3415 | Regression_eta: 0.0044 | Regression_pt: 0.3445 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1432 | Classification_binary: 0.2946 | Regression_cos_phi: 0.1261 | Regression_energy: 0.1781 | Regression_eta: 0.1207 | Regression_pt: 0.1868 | Regression_sin_phi: 0.1355 | LR: 9.81e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.7042s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0236 | max_forward: 0.8543 | max_loss: 0.1674 | max_backward: 2.1087 | max_optimizer_step: 0.3975 | max_tensorboard_logging: 0.5987 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5851.7 | uss_MB: 4653.7 | child_rss_MB: 6379.0 | allocated_MB: 446.7 | reserved_MB: 29402.0 | inactive_split_MB: 977.2 +rank7 [2026-10-28 15:14:30,754] INFO : Step 4800/50000 rank7 | Train Loss: 2.2522 | Train Loss Unweighted: 1.2790 | Train Loss Components: Auxiliary: 0.4197 | Cardinality: 0.2298 | Classification: 0.2444 | Classification_binary: 0.3099 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3546 | Regression_eta: 0.0050 | Regression_pt: 0.3598 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1470 | Classification_binary: 0.2810 | Regression_cos_phi: 0.2227 | Regression_energy: 0.1877 | Regression_eta: 0.1340 | Regression_pt: 0.1948 | Regression_sin_phi: 0.2108 | LR: 9.80e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4401s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7679 | max_loss: 0.2430 | max_backward: 1.9937 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0247 | Memory: rss_MB: 5851.1 | uss_MB: 4652.6 | child_rss_MB: 6380.8 | allocated_MB: 449.2 | reserved_MB: 29402.0 | inactive_split_MB: 1079.3 +rank7 [2026-10-28 15:19:27,570] INFO : Step 4900/50000 rank7 | Train Loss: 2.3477 | Train Loss Unweighted: 1.3387 | Train Loss Components: Auxiliary: 0.4345 | Cardinality: 0.2184 | Classification: 0.2519 | Classification_binary: 0.3791 | HitGrouping: 0.2376 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3456 | Regression_eta: 0.0053 | Regression_pt: 0.3513 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1318 | Classification_binary: 0.3416 | Regression_cos_phi: 0.1440 | Regression_energy: 0.1745 | Regression_eta: 0.1201 | Regression_pt: 0.1838 | Regression_sin_phi: 0.1512 | LR: 9.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5355s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0809 | max_forward: 0.8317 | max_loss: 0.1589 | max_backward: 1.9495 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.6002 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 5852.5 | uss_MB: 4655.4 | child_rss_MB: 6359.1 | allocated_MB: 450.0 | reserved_MB: 29402.0 | inactive_split_MB: 1364.8 +rank7 [2026-10-28 15:24:23,996] INFO : Step 5000/50000 rank7 | Train Loss: 2.2286 | Train Loss Unweighted: 1.2644 | Train Loss Components: Auxiliary: 0.4234 | Cardinality: 0.2722 | Classification: 0.2231 | Classification_binary: 0.3281 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3524 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.3101 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1773 | Regression_eta: 0.0933 | Regression_pt: 0.1803 | Regression_sin_phi: 0.1206 | LR: 9.78e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4826s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7723 | max_loss: 0.2461 | max_backward: 1.9456 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.6146 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 5845.4 | uss_MB: 4653.0 | child_rss_MB: 6371.7 | allocated_MB: 449.0 | reserved_MB: 29402.0 | inactive_split_MB: 1172.0 +rank7 [2026-10-28 15:24:24,982] INFO : Running validation on rank7 +rank7 [2026-10-28 15:24:24,982] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 15:24:24,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 15:25:19,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:25:19,570] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 15:25:44,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 15:25:45,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 15:25:46,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 15:25:47,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 15:25:58,006] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 15:25:58,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:25:59,250] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 15:26:15,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 15:26:16,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 15:26:17,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 15:26:18,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 15:26:28,451] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 15:26:28,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:26:29,409] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 15:26:44,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 15:26:45,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 15:26:46,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 15:26:47,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_5000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 15:26:57,391] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 15:32:01,929] INFO : Step 5100/50000 rank7 | Train Loss: 1.9847 | Train Loss Unweighted: 1.1699 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1930 | Classification: 0.2015 | Classification_binary: 0.2773 | HitGrouping: 0.2313 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3407 | Regression_eta: 0.0044 | Regression_pt: 0.3420 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1204 | Classification_binary: 0.2646 | Regression_cos_phi: 0.1141 | Regression_energy: 0.1535 | Regression_eta: 0.1181 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0872 | LR: 9.77e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.6409s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7577 | max_loss: 0.2085 | max_backward: 3.6716 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 1.1371 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5899.4 | uss_MB: 4699.9 | child_rss_MB: 6388.7 | allocated_MB: 446.2 | reserved_MB: 25176.0 | inactive_split_MB: 1180.5 +rank7 [2026-10-28 15:36:53,511] INFO : Step 5200/50000 rank7 | Train Loss: 2.2335 | Train Loss Unweighted: 1.2966 | Train Loss Components: Auxiliary: 0.4116 | Cardinality: 0.2101 | Classification: 0.2513 | Classification_binary: 0.3457 | HitGrouping: 0.2338 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3413 | Regression_eta: 0.0049 | Regression_pt: 0.3483 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 0.3479 | Regression_cos_phi: 0.2716 | Regression_energy: 0.1765 | Regression_eta: 0.1486 | Regression_pt: 0.1846 | Regression_sin_phi: 0.2468 | LR: 9.76e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.6374s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8344 | max_loss: 0.1476 | max_backward: 2.0477 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9777 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0169 | Memory: rss_MB: 5911.9 | uss_MB: 4199.4 | child_rss_MB: 6361.8 | allocated_MB: 449.2 | reserved_MB: 29212.0 | inactive_split_MB: 927.0 +rank7 [2026-10-28 15:41:36,526] INFO : Step 5300/50000 rank7 | Train Loss: 2.2120 | Train Loss Unweighted: 1.2824 | Train Loss Components: Auxiliary: 0.4208 | Cardinality: 0.2246 | Classification: 0.2237 | Classification_binary: 0.3424 | HitGrouping: 0.2324 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3517 | Regression_eta: 0.0044 | Regression_pt: 0.3555 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.3059 | Regression_cos_phi: 0.1596 | Regression_energy: 0.1976 | Regression_eta: 0.0890 | Regression_pt: 0.2087 | Regression_sin_phi: 0.1553 | LR: 9.75e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9244s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0470 | max_forward: 0.8048 | max_loss: 0.1497 | max_backward: 1.9807 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.0823 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5900.4 | uss_MB: 4701.4 | child_rss_MB: 6375.9 | allocated_MB: 450.2 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank7 [2026-10-28 15:46:33,436] INFO : Step 5400/50000 rank7 | Train Loss: 2.0323 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2039 | Classification: 0.2021 | Classification_binary: 0.3112 | HitGrouping: 0.2272 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2961 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1313 | Classification_binary: 0.3309 | Regression_cos_phi: 0.2409 | Regression_energy: 0.1518 | Regression_eta: 0.1452 | Regression_pt: 0.1570 | Regression_sin_phi: 0.3023 | LR: 9.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4647s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8661 | max_loss: 0.1428 | max_backward: 2.0384 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5905.8 | uss_MB: 4709.5 | child_rss_MB: 6379.9 | allocated_MB: 446.3 | reserved_MB: 29216.0 | inactive_split_MB: 1192.1 +rank7 [2026-10-28 15:51:26,877] INFO : Step 5500/50000 rank7 | Train Loss: 2.0969 | Train Loss Unweighted: 1.2334 | Train Loss Components: Auxiliary: 0.3952 | Cardinality: 0.1824 | Classification: 0.2155 | Classification_binary: 0.3160 | HitGrouping: 0.2328 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3440 | Regression_eta: 0.0044 | Regression_pt: 0.3488 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1481 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1818 | Regression_energy: 0.1770 | Regression_eta: 0.1573 | Regression_pt: 0.1841 | Regression_sin_phi: 0.1859 | LR: 9.73e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.5333s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0061 | max_forward: 1.1960 | max_loss: 0.2397 | max_backward: 2.6646 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 1.1026 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 5904.5 | uss_MB: 4707.0 | child_rss_MB: 6373.6 | allocated_MB: 445.7 | reserved_MB: 29216.0 | inactive_split_MB: 1206.5 +rank7 [2026-10-28 15:56:25,711] INFO : Step 5600/50000 rank7 | Train Loss: 2.5064 | Train Loss Unweighted: 1.3587 | Train Loss Components: Auxiliary: 0.4446 | Cardinality: 0.3323 | Classification: 0.2594 | Classification_binary: 0.4025 | HitGrouping: 0.2325 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3397 | Regression_eta: 0.0054 | Regression_pt: 0.3460 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.4078 | Regression_cos_phi: 0.2680 | Regression_energy: 0.1736 | Regression_eta: 0.1590 | Regression_pt: 0.1810 | Regression_sin_phi: 0.2298 | LR: 9.72e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.3184s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0093 | max_forward: 1.2355 | max_loss: 0.1515 | max_backward: 2.3309 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.9079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5907.4 | uss_MB: 4711.3 | child_rss_MB: 6374.7 | allocated_MB: 450.1 | reserved_MB: 29216.0 | inactive_split_MB: 999.5 +rank7 [2026-10-28 16:01:12,241] INFO : Step 5700/50000 rank7 | Train Loss: 1.8976 | Train Loss Unweighted: 1.1283 | Train Loss Components: Auxiliary: 0.3596 | Cardinality: 0.1704 | Classification: 0.1886 | Classification_binary: 0.3001 | HitGrouping: 0.2211 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3145 | Regression_eta: 0.0038 | Regression_pt: 0.3173 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1377 | Classification_binary: 0.3494 | Regression_cos_phi: 0.1667 | Regression_energy: 0.1858 | Regression_eta: 0.1255 | Regression_pt: 0.1927 | Regression_sin_phi: 0.1577 | LR: 9.71e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.5502s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0154 | max_forward: 0.7360 | max_loss: 0.2350 | max_backward: 1.8002 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9122 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 5901.9 | uss_MB: 4709.0 | child_rss_MB: 6382.7 | allocated_MB: 448.1 | reserved_MB: 29216.0 | inactive_split_MB: 1009.5 +rank7 [2026-10-28 16:06:09,458] INFO : Step 5800/50000 rank7 | Train Loss: 2.0691 | Train Loss Unweighted: 1.1926 | Train Loss Components: Auxiliary: 0.3849 | Cardinality: 0.1959 | Classification: 0.2235 | Classification_binary: 0.2928 | HitGrouping: 0.2302 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3314 | Regression_eta: 0.0045 | Regression_pt: 0.3358 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1090 | Classification_binary: 0.3045 | Regression_cos_phi: 0.1509 | Regression_energy: 0.1732 | Regression_eta: 0.1055 | Regression_pt: 0.1786 | Regression_sin_phi: 0.1405 | LR: 9.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2273s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0068 | max_forward: 1.2210 | max_loss: 0.3416 | max_backward: 2.2973 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.8322 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5906.1 | uss_MB: 4711.1 | child_rss_MB: 6376.1 | allocated_MB: 447.3 | reserved_MB: 29216.0 | inactive_split_MB: 1189.7 +rank7 [2026-10-28 16:11:19,901] INFO : Step 5900/50000 rank7 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3106 | Train Loss Components: Auxiliary: 0.4240 | Cardinality: 0.2107 | Classification: 0.2524 | Classification_binary: 0.3542 | HitGrouping: 0.2299 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3439 | Regression_eta: 0.0054 | Regression_pt: 0.3490 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1701 | Classification_binary: 0.3566 | Regression_cos_phi: 0.2084 | Regression_energy: 0.1846 | Regression_eta: 0.1965 | Regression_pt: 0.1909 | Regression_sin_phi: 0.2524 | LR: 9.69e-04 | DataLoad Time: 0.0163s | Model Forward Time: 7.1475s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7998 | max_loss: 0.2509 | max_backward: 2.0189 | max_optimizer_step: 0.2575 | max_tensorboard_logging: 3.0448 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5906.5 | uss_MB: 4712.1 | child_rss_MB: 6376.6 | allocated_MB: 449.6 | reserved_MB: 29216.0 | inactive_split_MB: 942.4 +rank7 [2026-10-28 16:16:40,508] INFO : Step 6000/50000 rank7 | Train Loss: 2.1980 | Train Loss Unweighted: 1.2174 | Train Loss Components: Auxiliary: 0.4062 | Cardinality: 0.2500 | Classification: 0.2266 | Classification_binary: 0.3412 | HitGrouping: 0.2284 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3187 | Regression_eta: 0.0048 | Regression_pt: 0.3212 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1205 | Classification_binary: 0.3441 | Regression_cos_phi: 0.1870 | Regression_energy: 0.1540 | Regression_eta: 0.1257 | Regression_pt: 0.1575 | Regression_sin_phi: 0.1783 | LR: 9.68e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.4976s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 1.0550 | max_loss: 0.1800 | max_backward: 1.7766 | max_optimizer_step: 0.2626 | max_tensorboard_logging: 0.6166 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5905.6 | uss_MB: 4709.8 | child_rss_MB: 6375.5 | allocated_MB: 446.1 | reserved_MB: 29960.0 | inactive_split_MB: 1161.8 +rank7 [2026-10-28 16:16:41,624] INFO : Running validation on rank7 +rank7 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:16:41,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:17:32,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:17:33,087] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 16:17:59,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:18:00,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:18:01,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:18:02,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:18:12,646] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:18:13,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:18:13,875] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 16:18:30,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:18:31,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:18:32,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:18:33,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:18:43,382] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:18:43,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:18:43,953] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 16:18:59,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:19:01,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:19:02,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:19:03,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_6000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:19:13,837] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 16:24:23,643] INFO : Step 6100/50000 rank7 | Train Loss: 1.8823 | Train Loss Unweighted: 1.0804 | Train Loss Components: Auxiliary: 0.3587 | Cardinality: 0.1831 | Classification: 0.1994 | Classification_binary: 0.2501 | HitGrouping: 0.2214 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3102 | Regression_eta: 0.0040 | Regression_pt: 0.3126 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1123 | Classification_binary: 0.2246 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1492 | Regression_eta: 0.0751 | Regression_pt: 0.1578 | Regression_sin_phi: 0.1658 | LR: 9.67e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9554s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7471 | max_loss: 0.2415 | max_backward: 1.8452 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.1237 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 6000.4 | uss_MB: 4799.0 | child_rss_MB: 6361.1 | allocated_MB: 445.0 | reserved_MB: 24188.0 | inactive_split_MB: 1042.2 +rank7 [2026-10-28 16:29:26,889] INFO : Step 6200/50000 rank7 | Train Loss: 2.0488 | Train Loss Unweighted: 1.1795 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.2037 | Classification: 0.2303 | Classification_binary: 0.3001 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3165 | Regression_eta: 0.0044 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1317 | Classification_binary: 0.3378 | Regression_cos_phi: 0.1406 | Regression_energy: 0.1720 | Regression_eta: 0.1101 | Regression_pt: 0.1795 | Regression_sin_phi: 0.1775 | LR: 9.66e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0905s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7382 | max_loss: 0.1533 | max_backward: 1.7781 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.8207 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0152 | Memory: rss_MB: 6005.6 | uss_MB: 3745.1 | child_rss_MB: 6355.8 | allocated_MB: 447.2 | reserved_MB: 24352.0 | inactive_split_MB: 1431.7 +rank7 [2026-10-28 16:34:17,122] INFO : Step 6300/50000 rank7 | Train Loss: 2.1765 | Train Loss Unweighted: 1.2343 | Train Loss Components: Auxiliary: 0.4071 | Cardinality: 0.2488 | Classification: 0.2344 | Classification_binary: 0.3575 | HitGrouping: 0.2215 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3149 | Regression_eta: 0.0044 | Regression_pt: 0.3184 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.3734 | Regression_cos_phi: 0.1552 | Regression_energy: 0.1610 | Regression_eta: 0.1092 | Regression_pt: 0.1665 | Regression_sin_phi: 0.1541 | LR: 9.65e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0299s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7896 | max_loss: 0.1842 | max_backward: 1.8671 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.1054 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6002.4 | uss_MB: 4807.8 | child_rss_MB: 6349.2 | allocated_MB: 450.5 | reserved_MB: 26230.0 | inactive_split_MB: 1399.3 +rank7 [2026-10-28 16:39:14,720] INFO : Step 6400/50000 rank7 | Train Loss: 1.9650 | Train Loss Unweighted: 1.1349 | Train Loss Components: Auxiliary: 0.3727 | Cardinality: 0.2181 | Classification: 0.2020 | Classification_binary: 0.2899 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3161 | Regression_eta: 0.0038 | Regression_pt: 0.3192 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3355 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1448 | Regression_eta: 0.1188 | Regression_pt: 0.1497 | Regression_sin_phi: 0.1457 | LR: 9.64e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7751s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7817 | max_loss: 0.1514 | max_backward: 1.5834 | max_optimizer_step: 0.2938 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5962.7 | uss_MB: 4805.6 | child_rss_MB: 6202.1 | allocated_MB: 443.2 | reserved_MB: 26236.0 | inactive_split_MB: 1190.9 +rank7 [2026-10-28 16:43:58,053] INFO : Step 6500/50000 rank7 | Train Loss: 1.9597 | Train Loss Unweighted: 1.1145 | Train Loss Components: Auxiliary: 0.3658 | Cardinality: 0.2082 | Classification: 0.2029 | Classification_binary: 0.2806 | HitGrouping: 0.2234 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3077 | Regression_eta: 0.0038 | Regression_pt: 0.3152 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1448 | Classification_binary: 0.2802 | Regression_cos_phi: 0.2130 | Regression_energy: 0.1569 | Regression_eta: 0.1302 | Regression_pt: 0.1623 | Regression_sin_phi: 0.1554 | LR: 9.63e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5009s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0454 | max_forward: 0.7657 | max_loss: 0.1403 | max_backward: 1.8022 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.8177 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 5917.3 | uss_MB: 4807.7 | child_rss_MB: 5983.9 | allocated_MB: 447.7 | reserved_MB: 26236.0 | inactive_split_MB: 1186.8 +rank7 [2026-10-28 16:48:52,483] INFO : Step 6600/50000 rank7 | Train Loss: 2.1614 | Train Loss Unweighted: 1.2403 | Train Loss Components: Auxiliary: 0.4072 | Cardinality: 0.2186 | Classification: 0.2173 | Classification_binary: 0.3619 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3245 | Regression_eta: 0.0046 | Regression_pt: 0.3274 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1220 | Classification_binary: 0.3424 | Regression_cos_phi: 0.1488 | Regression_energy: 0.1879 | Regression_eta: 0.1058 | Regression_pt: 0.1935 | Regression_sin_phi: 0.1276 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8073s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8036 | max_loss: 0.2561 | max_backward: 2.0021 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.8779 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 5921.8 | uss_MB: 4810.8 | child_rss_MB: 5982.0 | allocated_MB: 447.1 | reserved_MB: 26236.0 | inactive_split_MB: 1222.7 +rank7 [2026-10-28 16:53:47,413] INFO : Step 6700/50000 rank7 | Train Loss: 1.9396 | Train Loss Unweighted: 1.1172 | Train Loss Components: Auxiliary: 0.3640 | Cardinality: 0.1921 | Classification: 0.1931 | Classification_binary: 0.3060 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3039 | Regression_eta: 0.0042 | Regression_pt: 0.3059 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1005 | Classification_binary: 0.2581 | Regression_cos_phi: 0.1323 | Regression_energy: 0.1485 | Regression_eta: 0.1123 | Regression_pt: 0.1561 | Regression_sin_phi: 0.1367 | LR: 9.61e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.3354s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.9178 | max_loss: 0.1880 | max_backward: 2.3464 | max_optimizer_step: 0.2709 | max_tensorboard_logging: 0.9035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 5842.9 | uss_MB: 3729.8 | child_rss_MB: 5598.8 | allocated_MB: 449.3 | reserved_MB: 26624.0 | inactive_split_MB: 1246.9 +rank7 [2026-10-28 16:58:34,481] INFO : Step 6800/50000 rank7 | Train Loss: 1.9215 | Train Loss Unweighted: 1.0997 | Train Loss Components: Auxiliary: 0.3623 | Cardinality: 0.2016 | Classification: 0.2122 | Classification_binary: 0.2681 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3046 | Regression_eta: 0.0038 | Regression_pt: 0.3067 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1449 | Classification_binary: 0.2855 | Regression_cos_phi: 0.1755 | Regression_energy: 0.1781 | Regression_eta: 0.1133 | Regression_pt: 0.1847 | Regression_sin_phi: 0.1866 | LR: 9.60e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4635s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8318 | max_loss: 0.1502 | max_backward: 1.6649 | max_optimizer_step: 0.2195 | max_tensorboard_logging: 0.8217 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5844.2 | uss_MB: 4807.8 | child_rss_MB: 5561.0 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1440.9 +rank7 [2026-10-28 17:03:31,773] INFO : Step 6900/50000 rank7 | Train Loss: 2.0807 | Train Loss Unweighted: 1.1757 | Train Loss Components: Auxiliary: 0.3811 | Cardinality: 0.1994 | Classification: 0.2310 | Classification_binary: 0.3325 | HitGrouping: 0.2199 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2999 | Regression_eta: 0.0046 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1360 | Classification_binary: 0.3592 | Regression_cos_phi: 0.2294 | Regression_energy: 0.1665 | Regression_eta: 0.1429 | Regression_pt: 0.1734 | Regression_sin_phi: 0.2265 | LR: 9.58e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6881s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8275 | max_loss: 0.1499 | max_backward: 1.9490 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.8294 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5839.9 | uss_MB: 4811.2 | child_rss_MB: 5565.3 | allocated_MB: 447.6 | reserved_MB: 28676.0 | inactive_split_MB: 1395.2 +rank7 [2026-10-28 17:08:29,914] INFO : Step 7000/50000 rank7 | Train Loss: 2.2196 | Train Loss Unweighted: 1.2293 | Train Loss Components: Auxiliary: 0.3955 | Cardinality: 0.2350 | Classification: 0.2350 | Classification_binary: 0.3523 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3155 | Regression_eta: 0.0051 | Regression_pt: 0.3161 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1348 | Classification_binary: 0.3879 | Regression_cos_phi: 0.2095 | Regression_energy: 0.1667 | Regression_eta: 0.1792 | Regression_pt: 0.1741 | Regression_sin_phi: 0.1851 | LR: 9.57e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9231s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0611 | max_forward: 0.8455 | max_loss: 0.1584 | max_backward: 1.9494 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 1.0206 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 5827.8 | uss_MB: 4810.9 | child_rss_MB: 5516.4 | allocated_MB: 448.2 | reserved_MB: 28676.0 | inactive_split_MB: 1051.9 +rank7 [2026-10-28 17:08:30,791] INFO : Running validation on rank7 +rank7 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 17:08:30,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 17:09:23,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:09:24,266] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 17:09:52,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 17:09:53,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 17:09:55,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 17:09:57,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 17:10:07,041] INFO : Time taken to make predictions on device 7 is: 0.29 min +rank7 [2026-10-28 17:10:07,736] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:10:08,262] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 17:10:23,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 17:10:24,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 17:10:25,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 17:10:26,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 17:10:36,858] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 17:10:37,866] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:10:38,451] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 17:10:53,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 17:10:54,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 17:10:55,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 17:10:56,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_7000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 17:11:06,413] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 17:16:27,595] INFO : Step 7100/50000 rank7 | Train Loss: 1.7539 | Train Loss Unweighted: 1.0152 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.1626 | Classification: 0.1706 | Classification_binary: 0.2671 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2828 | Regression_eta: 0.0036 | Regression_pt: 0.2875 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2792 | Regression_cos_phi: 0.1094 | Regression_energy: 0.1417 | Regression_eta: 0.1085 | Regression_pt: 0.1525 | Regression_sin_phi: 0.0989 | LR: 9.56e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1472s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.8873 | max_loss: 0.4733 | max_backward: 2.1489 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 1.1060 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 5928.9 | uss_MB: 4903.9 | child_rss_MB: 5532.2 | allocated_MB: 447.2 | reserved_MB: 25722.0 | inactive_split_MB: 861.6 +rank7 [2026-10-28 17:21:34,077] INFO : Step 7200/50000 rank7 | Train Loss: 1.9621 | Train Loss Unweighted: 1.0889 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.2061 | Classification: 0.2066 | Classification_binary: 0.3151 | HitGrouping: 0.2184 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2780 | Regression_eta: 0.0039 | Regression_pt: 0.2808 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.3579 | Regression_cos_phi: 0.1386 | Regression_energy: 0.1545 | Regression_eta: 0.1243 | Regression_pt: 0.1613 | Regression_sin_phi: 0.1828 | LR: 9.55e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3462s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.8060 | max_loss: 0.1485 | max_backward: 1.8884 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5936.4 | uss_MB: 4915.4 | child_rss_MB: 5540.6 | allocated_MB: 451.5 | reserved_MB: 30170.0 | inactive_split_MB: 1411.3 +rank7 [2026-10-28 17:26:34,942] INFO : Step 7300/50000 rank7 | Train Loss: 2.1256 | Train Loss Unweighted: 1.1343 | Train Loss Components: Auxiliary: 0.3864 | Cardinality: 0.2846 | Classification: 0.2036 | Classification_binary: 0.3535 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2827 | Regression_eta: 0.0045 | Regression_pt: 0.2856 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1071 | Classification_binary: 0.3546 | Regression_cos_phi: 0.1731 | Regression_energy: 0.1544 | Regression_eta: 0.1269 | Regression_pt: 0.1580 | Regression_sin_phi: 0.1818 | LR: 9.53e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.4035s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.1152 | max_forward: 0.8829 | max_loss: 0.1602 | max_backward: 1.7447 | max_optimizer_step: 0.1967 | max_tensorboard_logging: 0.6150 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 5932.8 | uss_MB: 3780.8 | child_rss_MB: 5538.5 | allocated_MB: 449.2 | reserved_MB: 31610.0 | inactive_split_MB: 1328.6 +rank7 [2026-10-28 17:31:34,683] INFO : Step 7400/50000 rank7 | Train Loss: 1.7704 | Train Loss Unweighted: 1.0341 | Train Loss Components: Auxiliary: 0.3343 | Cardinality: 0.1759 | Classification: 0.1572 | Classification_binary: 0.2732 | HitGrouping: 0.2167 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2974 | Regression_eta: 0.0033 | Regression_pt: 0.2994 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2370 | Regression_cos_phi: 0.1161 | Regression_energy: 0.1565 | Regression_eta: 0.0791 | Regression_pt: 0.1606 | Regression_sin_phi: 0.1055 | LR: 9.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5944s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7337 | max_loss: 0.4768 | max_backward: 1.7808 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9770 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5919.5 | uss_MB: 4913.6 | child_rss_MB: 5490.6 | allocated_MB: 445.0 | reserved_MB: 31610.0 | inactive_split_MB: 1314.9 +rank7 [2026-10-28 17:36:30,885] INFO : Step 7500/50000 rank7 | Train Loss: 1.8306 | Train Loss Unweighted: 1.0305 | Train Loss Components: Auxiliary: 0.3498 | Cardinality: 0.1978 | Classification: 0.1725 | Classification_binary: 0.2588 | HitGrouping: 0.2097 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2927 | Regression_eta: 0.0035 | Regression_pt: 0.2989 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1307 | Classification_binary: 0.2783 | Regression_cos_phi: 0.1693 | Regression_energy: 0.1737 | Regression_eta: 0.1081 | Regression_pt: 0.1808 | Regression_sin_phi: 0.1739 | LR: 9.51e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6600s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8056 | max_loss: 0.1449 | max_backward: 1.9776 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8264 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 5884.3 | uss_MB: 4916.7 | child_rss_MB: 5302.1 | allocated_MB: 447.6 | reserved_MB: 33626.0 | inactive_split_MB: 1319.3 +rank7 [2026-10-28 17:41:09,112] INFO : Step 7600/50000 rank7 | Train Loss: 2.1052 | Train Loss Unweighted: 1.1776 | Train Loss Components: Auxiliary: 0.3953 | Cardinality: 0.2375 | Classification: 0.2247 | Classification_binary: 0.3422 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2988 | Regression_eta: 0.0044 | Regression_pt: 0.3029 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1274 | Classification_binary: 0.3723 | Regression_cos_phi: 0.1538 | Regression_energy: 0.1606 | Regression_eta: 0.1193 | Regression_pt: 0.1691 | Regression_sin_phi: 0.1698 | LR: 9.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7899s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0399 | max_forward: 0.8044 | max_loss: 0.1490 | max_backward: 1.9458 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.9344 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 5876.2 | uss_MB: 4916.0 | child_rss_MB: 5247.1 | allocated_MB: 449.5 | reserved_MB: 33626.0 | inactive_split_MB: 1326.9 +rank7 [2026-10-28 17:46:03,242] INFO : Step 7700/50000 rank7 | Train Loss: 1.9728 | Train Loss Unweighted: 1.0481 | Train Loss Components: Auxiliary: 0.3706 | Cardinality: 0.2931 | Classification: 0.1686 | Classification_binary: 0.3130 | HitGrouping: 0.2139 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2783 | Regression_eta: 0.0038 | Regression_pt: 0.2807 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.3086 | Regression_cos_phi: 0.1094 | Regression_energy: 0.1466 | Regression_eta: 0.1437 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1493 | LR: 9.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7825s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0072 | max_forward: 0.8095 | max_loss: 0.1456 | max_backward: 1.9736 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.9503 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5880.5 | uss_MB: 4917.5 | child_rss_MB: 5237.9 | allocated_MB: 447.8 | reserved_MB: 33626.0 | inactive_split_MB: 985.8 +rank7 [2026-10-28 17:51:02,466] INFO : Step 7800/50000 rank7 | Train Loss: 1.8130 | Train Loss Unweighted: 1.0032 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.2152 | Classification: 0.1891 | Classification_binary: 0.2566 | HitGrouping: 0.2159 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2750 | Regression_eta: 0.0035 | Regression_pt: 0.2753 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1060 | Classification_binary: 0.2646 | Regression_cos_phi: 0.1551 | Regression_energy: 0.1366 | Regression_eta: 0.1035 | Regression_pt: 0.1420 | Regression_sin_phi: 0.1429 | LR: 9.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4018s | Phase Times: max_pre_step_barrier: 0.1146 | max_batch_to_device: 0.0067 | max_forward: 0.7710 | max_loss: 0.2477 | max_backward: 1.9500 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.4751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0809 | Memory: rss_MB: 5878.0 | uss_MB: 4915.5 | child_rss_MB: 5245.9 | allocated_MB: 445.6 | reserved_MB: 33626.0 | inactive_split_MB: 1168.7 +rank7 [2026-10-28 17:56:05,210] INFO : Step 7900/50000 rank7 | Train Loss: 1.9904 | Train Loss Unweighted: 1.1333 | Train Loss Components: Auxiliary: 0.3731 | Cardinality: 0.1968 | Classification: 0.2138 | Classification_binary: 0.3291 | HitGrouping: 0.2169 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2893 | Regression_eta: 0.0042 | Regression_pt: 0.2926 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1537 | Classification_binary: 0.3159 | Regression_cos_phi: 0.1926 | Regression_energy: 0.1567 | Regression_eta: 0.1271 | Regression_pt: 0.1617 | Regression_sin_phi: 0.1762 | LR: 9.46e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.3972s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0074 | max_forward: 1.8170 | max_loss: 0.6708 | max_backward: 3.9218 | max_optimizer_step: 0.1968 | max_tensorboard_logging: 0.4745 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 5876.8 | uss_MB: 4917.5 | child_rss_MB: 5269.7 | allocated_MB: 448.2 | reserved_MB: 33626.0 | inactive_split_MB: 1231.2 +rank7 [2026-10-28 18:00:55,277] INFO : Step 8000/50000 rank7 | Train Loss: 1.9491 | Train Loss Unweighted: 1.0841 | Train Loss Components: Auxiliary: 0.3561 | Cardinality: 0.1916 | Classification: 0.1951 | Classification_binary: 0.3200 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2789 | Regression_eta: 0.0045 | Regression_pt: 0.2812 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.3037 | Regression_cos_phi: 0.1455 | Regression_energy: 0.1514 | Regression_eta: 0.1172 | Regression_pt: 0.1559 | Regression_sin_phi: 0.1504 | LR: 9.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4825s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0396 | max_forward: 0.7990 | max_loss: 0.4815 | max_backward: 1.9671 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5079 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0310 | Memory: rss_MB: 5875.7 | uss_MB: 4915.6 | child_rss_MB: 5266.8 | allocated_MB: 447.3 | reserved_MB: 33626.0 | inactive_split_MB: 1233.2 +rank7 [2026-10-28 18:00:56,228] INFO : Running validation on rank7 +rank7 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:00:56,228] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:01:47,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:01:48,314] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 18:02:14,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:02:15,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:02:16,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:02:17,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:02:27,916] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:02:28,339] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:02:28,858] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 18:02:44,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:02:45,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:02:46,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:02:47,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:02:57,608] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:02:58,281] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:02:58,745] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 18:03:14,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:03:15,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:03:15,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:03:16,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_8000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:03:26,725] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 18:08:30,021] INFO : Step 8100/50000 rank7 | Train Loss: 1.6702 | Train Loss Unweighted: 0.9551 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1649 | Classification: 0.1526 | Classification_binary: 0.2514 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2726 | Regression_eta: 0.0033 | Regression_pt: 0.2718 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1310 | Classification_binary: 0.2821 | Regression_cos_phi: 0.1947 | Regression_energy: 0.1499 | Regression_eta: 0.1048 | Regression_pt: 0.1532 | Regression_sin_phi: 0.1511 | LR: 9.43e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.3411s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7792 | max_loss: 0.1559 | max_backward: 1.8195 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.5289 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0575 | Memory: rss_MB: 7079.3 | uss_MB: 5729.7 | child_rss_MB: 5238.8 | allocated_MB: 445.1 | reserved_MB: 25558.0 | inactive_split_MB: 1188.9 +rank7 [2026-10-28 18:13:41,920] INFO : Step 8200/50000 rank7 | Train Loss: 1.7904 | Train Loss Unweighted: 1.0109 | Train Loss Components: Auxiliary: 0.3294 | Cardinality: 0.1623 | Classification: 0.1927 | Classification_binary: 0.2846 | HitGrouping: 0.2079 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2618 | Regression_eta: 0.0035 | Regression_pt: 0.2641 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1203 | Classification_binary: 0.3203 | Regression_cos_phi: 0.1918 | Regression_energy: 0.1617 | Regression_eta: 0.1162 | Regression_pt: 0.1667 | Regression_sin_phi: 0.1881 | LR: 9.42e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3247s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7780 | max_loss: 0.5105 | max_backward: 1.9203 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5977.5 | uss_MB: 5031.2 | child_rss_MB: 5263.8 | allocated_MB: 447.6 | reserved_MB: 25560.0 | inactive_split_MB: 1475.4 +rank7 [2026-10-28 18:18:39,532] INFO : Step 8300/50000 rank7 | Train Loss: 1.9419 | Train Loss Unweighted: 1.0717 | Train Loss Components: Auxiliary: 0.3579 | Cardinality: 0.2162 | Classification: 0.1876 | Classification_binary: 0.3351 | HitGrouping: 0.2071 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2689 | Regression_eta: 0.0042 | Regression_pt: 0.2719 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1070 | Classification_binary: 0.3366 | Regression_cos_phi: 0.1456 | Regression_energy: 0.1402 | Regression_eta: 0.1186 | Regression_pt: 0.1463 | Regression_sin_phi: 0.1258 | LR: 9.40e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6370s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0075 | max_forward: 1.0737 | max_loss: 0.1909 | max_backward: 2.2152 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4614 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 5969.7 | uss_MB: 5032.6 | child_rss_MB: 5258.7 | allocated_MB: 450.1 | reserved_MB: 25566.0 | inactive_split_MB: 1473.9 +rank7 [2026-10-28 18:23:43,088] INFO : Step 8400/50000 rank7 | Train Loss: 1.7180 | Train Loss Unweighted: 0.9680 | Train Loss Components: Auxiliary: 0.3147 | Cardinality: 0.1606 | Classification: 0.1659 | Classification_binary: 0.2792 | HitGrouping: 0.2065 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2569 | Regression_eta: 0.0037 | Regression_pt: 0.2585 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2850 | Regression_cos_phi: 0.1664 | Regression_energy: 0.1250 | Regression_eta: 0.0847 | Regression_pt: 0.1290 | Regression_sin_phi: 0.1038 | LR: 9.39e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.1818s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7234 | max_loss: 0.1439 | max_backward: 1.7656 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5964.7 | uss_MB: 5028.1 | child_rss_MB: 5235.4 | allocated_MB: 445.8 | reserved_MB: 27618.0 | inactive_split_MB: 1464.9 +rank7 [2026-10-28 18:28:52,138] INFO : Step 8500/50000 rank7 | Train Loss: 1.6830 | Train Loss Unweighted: 0.9196 | Train Loss Components: Auxiliary: 0.3105 | Cardinality: 0.1843 | Classification: 0.1643 | Classification_binary: 0.2464 | HitGrouping: 0.2069 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2499 | Regression_eta: 0.0034 | Regression_pt: 0.2520 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.2299 | Regression_cos_phi: 0.1914 | Regression_energy: 0.1396 | Regression_eta: 0.1102 | Regression_pt: 0.1459 | Regression_sin_phi: 0.1486 | LR: 9.37e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5042s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.8117 | max_loss: 0.5174 | max_backward: 1.9340 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.5082 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 5968.9 | uss_MB: 5031.7 | child_rss_MB: 5261.3 | allocated_MB: 448.6 | reserved_MB: 27618.0 | inactive_split_MB: 1349.9 +rank7 [2026-10-28 18:34:09,472] INFO : Step 8600/50000 rank7 | Train Loss: 1.8258 | Train Loss Unweighted: 1.0262 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.1744 | Classification: 0.1788 | Classification_binary: 0.3088 | HitGrouping: 0.2045 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2636 | Regression_eta: 0.0038 | Regression_pt: 0.2672 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1028 | Classification_binary: 0.3181 | Regression_cos_phi: 0.1064 | Regression_energy: 0.1333 | Regression_eta: 0.0924 | Regression_pt: 0.1373 | Regression_sin_phi: 0.1331 | LR: 9.36e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7742s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0839 | max_forward: 0.8384 | max_loss: 0.1529 | max_backward: 1.9425 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.8319 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5967.3 | uss_MB: 5032.4 | child_rss_MB: 5257.6 | allocated_MB: 449.0 | reserved_MB: 27618.0 | inactive_split_MB: 1457.7 +rank7 [2026-10-28 18:39:14,579] INFO : Step 8700/50000 rank7 | Train Loss: 1.8656 | Train Loss Unweighted: 0.9777 | Train Loss Components: Auxiliary: 0.3497 | Cardinality: 0.2571 | Classification: 0.1599 | Classification_binary: 0.2941 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2590 | Regression_eta: 0.0035 | Regression_pt: 0.2573 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2970 | Regression_cos_phi: 0.1625 | Regression_energy: 0.1381 | Regression_eta: 0.0872 | Regression_pt: 0.1394 | Regression_sin_phi: 0.1222 | LR: 9.34e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.7194s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 0.7881 | max_loss: 0.1517 | max_backward: 1.8585 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.9529 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 5967.0 | uss_MB: 5031.1 | child_rss_MB: 5260.3 | allocated_MB: 445.3 | reserved_MB: 27620.0 | inactive_split_MB: 1223.3 +rank7 [2026-10-28 18:44:19,414] INFO : Step 8800/50000 rank7 | Train Loss: 1.5510 | Train Loss Unweighted: 0.8539 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1562 | Classification: 0.1305 | Classification_binary: 0.2302 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2438 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.2242 | Regression_cos_phi: 0.1214 | Regression_energy: 0.1390 | Regression_eta: 0.0640 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0658 | LR: 9.33e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.4101s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 1.1353 | max_loss: 0.2847 | max_backward: 3.0276 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.9371 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 5959.1 | uss_MB: 3786.2 | child_rss_MB: 5233.8 | allocated_MB: 447.6 | reserved_MB: 27620.0 | inactive_split_MB: 1178.1 +rank7 [2026-10-28 18:49:28,517] INFO : Step 8900/50000 rank7 | Train Loss: 2.0208 | Train Loss Unweighted: 1.0418 | Train Loss Components: Auxiliary: 0.3691 | Cardinality: 0.2976 | Classification: 0.1802 | Classification_binary: 0.3114 | HitGrouping: 0.2094 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2684 | Regression_eta: 0.0039 | Regression_pt: 0.2736 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1120 | Classification_binary: 0.3010 | Regression_cos_phi: 0.1303 | Regression_energy: 0.1499 | Regression_eta: 0.1037 | Regression_pt: 0.1582 | Regression_sin_phi: 0.1697 | LR: 9.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7917s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8799 | max_loss: 0.1424 | max_backward: 2.0950 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 5959.1 | uss_MB: 5032.2 | child_rss_MB: 5262.2 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1467.4 +rank7 [2026-10-28 18:54:53,968] INFO : Step 9000/50000 rank7 | Train Loss: 1.8831 | Train Loss Unweighted: 1.0179 | Train Loss Components: Auxiliary: 0.3589 | Cardinality: 0.2353 | Classification: 0.1586 | Classification_binary: 0.3233 | HitGrouping: 0.2064 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2633 | Regression_eta: 0.0036 | Regression_pt: 0.2653 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1020 | Classification_binary: 0.3319 | Regression_cos_phi: 0.1666 | Regression_energy: 0.1558 | Regression_eta: 0.1043 | Regression_pt: 0.1610 | Regression_sin_phi: 0.1761 | LR: 9.30e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.8374s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0068 | max_forward: 0.9459 | max_loss: 0.4837 | max_backward: 2.2436 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.6358 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 5947.6 | uss_MB: 4181.9 | child_rss_MB: 5262.7 | allocated_MB: 448.7 | reserved_MB: 27620.0 | inactive_split_MB: 1226.7 +rank7 [2026-10-28 18:54:54,936] INFO : Running validation on rank7 +rank7 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:54:54,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:55:52,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:55:53,258] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 18:56:19,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:56:21,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:56:22,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:56:23,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:56:33,809] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-28 18:56:34,837] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:56:35,474] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 18:56:51,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:56:52,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:56:53,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:56:54,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:57:04,751] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:57:05,415] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:57:05,962] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 18:57:21,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:57:21,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:57:22,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:57:23,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_9000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:57:33,558] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 19:03:08,598] INFO : Step 9100/50000 rank7 | Train Loss: 1.5011 | Train Loss Unweighted: 0.8295 | Train Loss Components: Auxiliary: 0.2790 | Cardinality: 0.1451 | Classification: 0.1296 | Classification_binary: 0.2417 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2260 | Regression_eta: 0.0029 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.2285 | Regression_cos_phi: 0.1177 | Regression_energy: 0.1292 | Regression_eta: 0.0834 | Regression_pt: 0.1317 | Regression_sin_phi: 0.1127 | LR: 9.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.1696s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7384 | max_loss: 0.1441 | max_backward: 2.6967 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 1.5246 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 7023.3 | uss_MB: 6094.6 | child_rss_MB: 5245.9 | allocated_MB: 444.6 | reserved_MB: 23432.0 | inactive_split_MB: 1182.1 +rank7 [2026-10-28 19:08:18,222] INFO : Step 9200/50000 rank7 | Train Loss: 1.6850 | Train Loss Unweighted: 0.9372 | Train Loss Components: Auxiliary: 0.3167 | Cardinality: 0.1702 | Classification: 0.1510 | Classification_binary: 0.2613 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2577 | Regression_eta: 0.0035 | Regression_pt: 0.2602 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2869 | Regression_cos_phi: 0.1459 | Regression_energy: 0.1438 | Regression_eta: 0.1196 | Regression_pt: 0.1476 | Regression_sin_phi: 0.1316 | LR: 9.27e-04 | DataLoad Time: 0.0259s | Model Forward Time: 6.5430s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 1.2596 | max_loss: 0.2026 | max_backward: 2.9495 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6023.5 | uss_MB: 4366.4 | child_rss_MB: 5260.6 | allocated_MB: 448.2 | reserved_MB: 24058.0 | inactive_split_MB: 1125.4 +rank7 [2026-10-28 19:13:30,253] INFO : Step 9300/50000 rank7 | Train Loss: 1.8909 | Train Loss Unweighted: 1.0215 | Train Loss Components: Auxiliary: 0.3570 | Cardinality: 0.2325 | Classification: 0.1828 | Classification_binary: 0.3186 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2536 | Regression_eta: 0.0036 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1079 | Classification_binary: 0.3153 | Regression_cos_phi: 0.2033 | Regression_energy: 0.1282 | Regression_eta: 0.1368 | Regression_pt: 0.1334 | Regression_sin_phi: 0.1648 | LR: 9.25e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3918s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1063 | max_forward: 0.8940 | max_loss: 0.1524 | max_backward: 1.8613 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.4840 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6024.4 | uss_MB: 5099.1 | child_rss_MB: 5265.3 | allocated_MB: 451.2 | reserved_MB: 25436.0 | inactive_split_MB: 1216.0 +rank7 [2026-10-28 19:18:55,613] INFO : Step 9400/50000 rank7 | Train Loss: 1.6593 | Train Loss Unweighted: 0.9098 | Train Loss Components: Auxiliary: 0.3078 | Cardinality: 0.1596 | Classification: 0.1397 | Classification_binary: 0.2638 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2497 | Regression_eta: 0.0034 | Regression_pt: 0.2495 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2790 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1309 | Regression_eta: 0.0765 | Regression_pt: 0.1351 | Regression_sin_phi: 0.1432 | LR: 9.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2015s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.7465 | max_loss: 0.1445 | max_backward: 1.7313 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.5253 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6019.1 | uss_MB: 5096.2 | child_rss_MB: 5241.6 | allocated_MB: 447.7 | reserved_MB: 25438.0 | inactive_split_MB: 1132.5 +rank7 [2026-10-28 19:24:20,791] INFO : Step 9500/50000 rank7 | Train Loss: 1.6082 | Train Loss Unweighted: 0.8750 | Train Loss Components: Auxiliary: 0.3044 | Cardinality: 0.1728 | Classification: 0.1426 | Classification_binary: 0.2246 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2503 | Regression_eta: 0.0032 | Regression_pt: 0.2508 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.2061 | Regression_cos_phi: 0.1701 | Regression_energy: 0.1340 | Regression_eta: 0.0800 | Regression_pt: 0.1374 | Regression_sin_phi: 0.1000 | LR: 9.22e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.2835s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7372 | max_loss: 0.5100 | max_backward: 2.4158 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 1.0083 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 6019.1 | uss_MB: 5096.6 | child_rss_MB: 5236.6 | allocated_MB: 447.6 | reserved_MB: 25920.0 | inactive_split_MB: 1135.2 +rank7 [2026-10-28 19:29:34,440] INFO : Step 9600/50000 rank7 | Train Loss: 1.7674 | Train Loss Unweighted: 0.9635 | Train Loss Components: Auxiliary: 0.3311 | Cardinality: 0.2058 | Classification: 0.1641 | Classification_binary: 0.2880 | HitGrouping: 0.2017 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2497 | Regression_eta: 0.0032 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1044 | Classification_binary: 0.3067 | Regression_cos_phi: 0.1595 | Regression_energy: 0.1240 | Regression_eta: 0.0939 | Regression_pt: 0.1267 | Regression_sin_phi: 0.1617 | LR: 9.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6059s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0211 | max_forward: 0.7585 | max_loss: 0.4966 | max_backward: 2.7798 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9248 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6015.5 | uss_MB: 5098.3 | child_rss_MB: 5264.4 | allocated_MB: 448.7 | reserved_MB: 27284.0 | inactive_split_MB: 1187.3 +rank7 [2026-10-28 19:34:53,791] INFO : Step 9700/50000 rank7 | Train Loss: 1.7176 | Train Loss Unweighted: 0.9285 | Train Loss Components: Auxiliary: 0.3243 | Cardinality: 0.2088 | Classification: 0.1463 | Classification_binary: 0.3022 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2356 | Regression_eta: 0.0032 | Regression_pt: 0.2379 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.3105 | Regression_cos_phi: 0.1231 | Regression_energy: 0.1271 | Regression_eta: 0.0802 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1468 | LR: 9.19e-04 | DataLoad Time: 0.0118s | Model Forward Time: 4.9683s | Phase Times: max_pre_step_barrier: 0.0435 | max_batch_to_device: 0.0932 | max_forward: 0.8645 | max_loss: 0.1552 | max_backward: 2.0116 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0373 | Memory: rss_MB: 6014.4 | uss_MB: 5100.9 | child_rss_MB: 5261.9 | allocated_MB: 449.3 | reserved_MB: 27284.0 | inactive_split_MB: 1184.9 +rank7 [2026-10-28 19:39:53,936] INFO : Step 9800/50000 rank7 | Train Loss: 1.4908 | Train Loss Unweighted: 0.8301 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1622 | Classification: 0.1351 | Classification_binary: 0.2234 | HitGrouping: 0.1995 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2319 | Regression_eta: 0.0027 | Regression_pt: 0.2340 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2086 | Regression_cos_phi: 0.1286 | Regression_energy: 0.1410 | Regression_eta: 0.0804 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0966 | LR: 9.17e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.5605s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.7639 | max_loss: 0.1458 | max_backward: 1.7436 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.8392 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6002.1 | uss_MB: 5097.7 | child_rss_MB: 5234.5 | allocated_MB: 448.9 | reserved_MB: 27972.0 | inactive_split_MB: 1188.9 +rank7 [2026-10-28 19:44:46,546] INFO : Step 9900/50000 rank7 | Train Loss: 1.6370 | Train Loss Unweighted: 0.8736 | Train Loss Components: Auxiliary: 0.3071 | Cardinality: 0.1812 | Classification: 0.1419 | Classification_binary: 0.2557 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2333 | Regression_eta: 0.0032 | Regression_pt: 0.2358 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2381 | Regression_cos_phi: 0.1214 | Regression_energy: 0.1275 | Regression_eta: 0.0709 | Regression_pt: 0.1315 | Regression_sin_phi: 0.1192 | LR: 9.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9295s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8024 | max_loss: 0.1478 | max_backward: 2.0105 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.9325 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6003.1 | uss_MB: 5091.7 | child_rss_MB: 5261.2 | allocated_MB: 448.1 | reserved_MB: 29300.0 | inactive_split_MB: 1161.2 +rank7 [2026-10-28 19:49:37,630] INFO : Step 10000/50000 rank7 | Train Loss: 1.7651 | Train Loss Unweighted: 0.9525 | Train Loss Components: Auxiliary: 0.3266 | Cardinality: 0.1933 | Classification: 0.1606 | Classification_binary: 0.3104 | HitGrouping: 0.1938 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2353 | Regression_eta: 0.0036 | Regression_pt: 0.2387 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1156 | Classification_binary: 0.3123 | Regression_cos_phi: 0.1401 | Regression_energy: 0.1032 | Regression_eta: 0.0919 | Regression_pt: 0.1064 | Regression_sin_phi: 0.1583 | LR: 9.14e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.6240s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8479 | max_loss: 0.2561 | max_backward: 2.4598 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.1725 | max_tensorboard_flush: 0.0167 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5992.4 | uss_MB: 4100.6 | child_rss_MB: 5266.8 | allocated_MB: 449.6 | reserved_MB: 29300.0 | inactive_split_MB: 1054.8 +rank7 [2026-10-28 19:49:38,691] INFO : Running validation on rank7 +rank7 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 19:49:38,691] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 19:50:35,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:50:35,625] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 19:51:01,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 19:51:02,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 19:51:03,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 19:51:05,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 19:51:15,016] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:51:15,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:51:16,369] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 19:51:32,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 19:51:33,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 19:51:34,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 19:51:35,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 19:51:45,501] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:51:46,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:51:46,686] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 19:52:02,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 19:52:03,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 19:52:04,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 19:52:04,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_10000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 19:52:14,850] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 19:57:27,968] INFO : Step 10100/50000 rank7 | Train Loss: 1.5566 | Train Loss Unweighted: 0.8294 | Train Loss Components: Auxiliary: 0.2887 | Cardinality: 0.1904 | Classification: 0.1212 | Classification_binary: 0.2554 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2218 | Regression_eta: 0.0028 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2337 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0909 | Regression_eta: 0.0790 | Regression_pt: 0.0927 | Regression_sin_phi: 0.1463 | LR: 9.12e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7465s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8431 | max_loss: 0.1475 | max_backward: 1.8773 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.8563 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0490 | Memory: rss_MB: 7043.1 | uss_MB: 6134.7 | child_rss_MB: 5263.6 | allocated_MB: 447.0 | reserved_MB: 27414.0 | inactive_split_MB: 1071.1 +rank7 [2026-10-28 20:02:15,542] INFO : Step 10200/50000 rank7 | Train Loss: 1.5727 | Train Loss Unweighted: 0.8491 | Train Loss Components: Auxiliary: 0.2967 | Cardinality: 0.1713 | Classification: 0.1352 | Classification_binary: 0.2392 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2325 | Regression_eta: 0.0033 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2477 | Regression_cos_phi: 0.1422 | Regression_energy: 0.1203 | Regression_eta: 0.1137 | Regression_pt: 0.1260 | Regression_sin_phi: 0.1502 | LR: 9.11e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6808s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8009 | max_loss: 0.1827 | max_backward: 2.0393 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.8075 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6067.3 | uss_MB: 5162.1 | child_rss_MB: 5259.8 | allocated_MB: 448.1 | reserved_MB: 27414.0 | inactive_split_MB: 1205.3 +rank7 [2026-10-28 20:07:31,753] INFO : Step 10300/50000 rank7 | Train Loss: 1.6696 | Train Loss Unweighted: 0.9084 | Train Loss Components: Auxiliary: 0.3133 | Cardinality: 0.1808 | Classification: 0.1430 | Classification_binary: 0.3005 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2288 | Regression_eta: 0.0032 | Regression_pt: 0.2294 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2820 | Regression_cos_phi: 0.1457 | Regression_energy: 0.1155 | Regression_eta: 0.0795 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1022 | LR: 9.09e-04 | DataLoad Time: 0.0237s | Model Forward Time: 5.1980s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0268 | max_forward: 1.0873 | max_loss: 0.1498 | max_backward: 2.1789 | max_optimizer_step: 0.2883 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6065.0 | uss_MB: 5165.0 | child_rss_MB: 5255.0 | allocated_MB: 453.2 | reserved_MB: 28022.0 | inactive_split_MB: 1015.7 +rank7 [2026-10-28 20:12:49,154] INFO : Step 10400/50000 rank7 | Train Loss: 1.5281 | Train Loss Unweighted: 0.8364 | Train Loss Components: Auxiliary: 0.2869 | Cardinality: 0.1569 | Classification: 0.1341 | Classification_binary: 0.2596 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2164 | Regression_eta: 0.0028 | Regression_pt: 0.2203 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2348 | Regression_cos_phi: 0.0918 | Regression_energy: 0.1110 | Regression_eta: 0.0751 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0943 | LR: 9.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2208s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7839 | max_loss: 0.1459 | max_backward: 2.3684 | max_optimizer_step: 0.1978 | max_tensorboard_logging: 0.6278 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 6060.3 | uss_MB: 4235.3 | child_rss_MB: 5232.5 | allocated_MB: 445.0 | reserved_MB: 28022.0 | inactive_split_MB: 966.6 +rank7 [2026-10-28 20:18:09,148] INFO : Step 10500/50000 rank7 | Train Loss: 1.3897 | Train Loss Unweighted: 0.7700 | Train Loss Components: Auxiliary: 0.2658 | Cardinality: 0.1540 | Classification: 0.1180 | Classification_binary: 0.2002 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2228 | Regression_eta: 0.0024 | Regression_pt: 0.2240 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0952 | Regression_eta: 0.0582 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0617 | LR: 9.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3220s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7376 | max_loss: 0.1406 | max_backward: 1.8229 | max_optimizer_step: 0.2050 | max_tensorboard_logging: 0.6391 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 6027.4 | uss_MB: 4417.6 | child_rss_MB: 5094.6 | allocated_MB: 447.1 | reserved_MB: 30058.0 | inactive_split_MB: 1186.9 +rank7 [2026-10-28 20:23:17,055] INFO : Step 10600/50000 rank7 | Train Loss: 1.5620 | Train Loss Unweighted: 0.8374 | Train Loss Components: Auxiliary: 0.2894 | Cardinality: 0.1780 | Classification: 0.1401 | Classification_binary: 0.2647 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2117 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.2637 | Regression_cos_phi: 0.1292 | Regression_energy: 0.1199 | Regression_eta: 0.0850 | Regression_pt: 0.1246 | Regression_sin_phi: 0.1187 | LR: 9.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0938s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8241 | max_loss: 0.1496 | max_backward: 2.1670 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.0219 | max_tensorboard_flush: 0.0015 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 5960.8 | uss_MB: 5163.5 | child_rss_MB: 4876.7 | allocated_MB: 451.7 | reserved_MB: 30058.0 | inactive_split_MB: 944.8 +rank7 [2026-10-28 20:28:22,709] INFO : Step 10700/50000 rank7 | Train Loss: 1.6551 | Train Loss Unweighted: 0.8977 | Train Loss Components: Auxiliary: 0.3174 | Cardinality: 0.2139 | Classification: 0.1436 | Classification_binary: 0.2815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2320 | Regression_eta: 0.0030 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.2584 | Regression_cos_phi: 0.0966 | Regression_energy: 0.1378 | Regression_eta: 0.0800 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0988 | LR: 9.02e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.8711s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 1.0142 | max_loss: 0.1516 | max_backward: 1.8697 | max_optimizer_step: 0.2017 | max_tensorboard_logging: 0.9746 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5946.3 | uss_MB: 5160.2 | child_rss_MB: 4877.1 | allocated_MB: 449.5 | reserved_MB: 30060.0 | inactive_split_MB: 1187.9 +rank7 [2026-10-28 20:33:28,996] INFO : Step 10800/50000 rank7 | Train Loss: 1.4049 | Train Loss Unweighted: 0.7304 | Train Loss Components: Auxiliary: 0.2639 | Cardinality: 0.1929 | Classification: 0.1015 | Classification_binary: 0.2090 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2072 | Regression_eta: 0.0025 | Regression_pt: 0.2076 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2415 | Regression_cos_phi: 0.1049 | Regression_energy: 0.1058 | Regression_eta: 0.0791 | Regression_pt: 0.1095 | Regression_sin_phi: 0.1107 | LR: 9.00e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.6821s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 1.0825 | max_loss: 0.1723 | max_backward: 1.7659 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.9751 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0350 | Memory: rss_MB: 5909.6 | uss_MB: 5149.8 | child_rss_MB: 4776.3 | allocated_MB: 446.2 | reserved_MB: 30060.0 | inactive_split_MB: 1005.8 +rank7 [2026-10-28 20:38:44,060] INFO : Step 10900/50000 rank7 | Train Loss: 1.4441 | Train Loss Unweighted: 0.7711 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1605 | Classification: 0.1280 | Classification_binary: 0.2331 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2005 | Regression_eta: 0.0026 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2491 | Regression_cos_phi: 0.0940 | Regression_energy: 0.1073 | Regression_eta: 0.0639 | Regression_pt: 0.1126 | Regression_sin_phi: 0.1321 | LR: 8.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7058s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7521 | max_loss: 0.2501 | max_backward: 1.9261 | max_optimizer_step: 0.3045 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5918.9 | uss_MB: 5159.1 | child_rss_MB: 4798.7 | allocated_MB: 449.2 | reserved_MB: 30060.0 | inactive_split_MB: 966.3 +rank7 [2026-10-28 20:43:46,526] INFO : Step 11000/50000 rank7 | Train Loss: 1.6462 | Train Loss Unweighted: 0.8747 | Train Loss Components: Auxiliary: 0.3099 | Cardinality: 0.2080 | Classification: 0.1486 | Classification_binary: 0.2945 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2103 | Regression_eta: 0.0031 | Regression_pt: 0.2149 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1168 | Classification_binary: 0.3105 | Regression_cos_phi: 0.1479 | Regression_energy: 0.1170 | Regression_eta: 0.1301 | Regression_pt: 0.1232 | Regression_sin_phi: 0.1691 | LR: 8.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4430s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7664 | max_loss: 0.1581 | max_backward: 1.7271 | max_optimizer_step: 0.3034 | max_tensorboard_logging: 0.6783 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5907.8 | uss_MB: 5151.3 | child_rss_MB: 4796.8 | allocated_MB: 449.1 | reserved_MB: 30060.0 | inactive_split_MB: 954.9 +rank7 [2026-10-28 20:43:47,416] INFO : Running validation on rank7 +rank7 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 20:43:47,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 20:44:41,297] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:44:41,885] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 20:45:09,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 20:45:10,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 20:45:11,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 20:45:12,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 20:45:22,941] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 20:45:23,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:45:24,126] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 20:45:40,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 20:45:41,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 20:45:42,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 20:45:44,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 20:45:54,530] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 20:45:55,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:45:55,828] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 20:46:11,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 20:46:12,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 20:46:13,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 20:46:14,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_11000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 20:46:24,984] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 20:51:55,950] INFO : Step 11100/50000 rank7 | Train Loss: 1.4042 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1517 | Classification: 0.1198 | Classification_binary: 0.2314 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1042 | Regression_energy: 0.1073 | Regression_eta: 0.0872 | Regression_pt: 0.1114 | Regression_sin_phi: 0.1261 | LR: 8.95e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8319s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7759 | max_loss: 0.1507 | max_backward: 1.7797 | max_optimizer_step: 0.2593 | max_tensorboard_logging: 0.9216 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 7212.5 | uss_MB: 4056.3 | child_rss_MB: 4795.4 | allocated_MB: 446.8 | reserved_MB: 26246.0 | inactive_split_MB: 1221.2 +rank7 [2026-10-28 20:57:10,090] INFO : Step 11200/50000 rank7 | Train Loss: 1.3286 | Train Loss Unweighted: 0.7108 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1623 | Classification: 0.1143 | Classification_binary: 0.1979 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1961 | Regression_eta: 0.0022 | Regression_pt: 0.1977 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0947 | Regression_energy: 0.1200 | Regression_eta: 0.0572 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0843 | LR: 8.93e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.8519s | Phase Times: max_pre_step_barrier: 0.1081 | max_batch_to_device: 0.0161 | max_forward: 0.9970 | max_loss: 0.1823 | max_backward: 2.0292 | max_optimizer_step: 0.2807 | max_tensorboard_logging: 0.7652 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6018.9 | uss_MB: 5260.4 | child_rss_MB: 4796.7 | allocated_MB: 447.6 | reserved_MB: 27642.0 | inactive_split_MB: 1006.0 +rank7 [2026-10-28 21:02:18,461] INFO : Step 11300/50000 rank7 | Train Loss: 1.5745 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2995 | Cardinality: 0.1776 | Classification: 0.1353 | Classification_binary: 0.2726 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2086 | Regression_eta: 0.0030 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1167 | Regression_energy: 0.1077 | Regression_eta: 0.0845 | Regression_pt: 0.1124 | Regression_sin_phi: 0.1190 | LR: 8.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.8347 | max_loss: 0.1507 | max_backward: 2.3640 | max_optimizer_step: 0.2242 | max_tensorboard_logging: 0.9295 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6018.5 | uss_MB: 5182.2 | child_rss_MB: 4794.3 | allocated_MB: 450.7 | reserved_MB: 27918.0 | inactive_split_MB: 1399.4 +rank7 [2026-10-28 21:07:20,244] INFO : Step 11400/50000 rank7 | Train Loss: 1.4680 | Train Loss Unweighted: 0.7894 | Train Loss Components: Auxiliary: 0.2778 | Cardinality: 0.1564 | Classification: 0.1207 | Classification_binary: 0.2626 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1993 | Regression_eta: 0.0026 | Regression_pt: 0.2011 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2541 | Regression_cos_phi: 0.1017 | Regression_energy: 0.1019 | Regression_eta: 0.0764 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0975 | LR: 8.89e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.9838s | Phase Times: max_pre_step_barrier: 0.0102 | max_batch_to_device: 0.0070 | max_forward: 0.9186 | max_loss: 0.1525 | max_backward: 2.1279 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.9149 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5988.3 | uss_MB: 5261.0 | child_rss_MB: 4708.7 | allocated_MB: 447.6 | reserved_MB: 27922.0 | inactive_split_MB: 1354.4 +rank7 [2026-10-28 21:12:15,937] INFO : Step 11500/50000 rank7 | Train Loss: 1.3515 | Train Loss Unweighted: 0.7426 | Train Loss Components: Auxiliary: 0.2558 | Cardinality: 0.1370 | Classification: 0.1204 | Classification_binary: 0.2027 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0023 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0945 | Classification_binary: 0.2150 | Regression_cos_phi: 0.1305 | Regression_energy: 0.1207 | Regression_eta: 0.0728 | Regression_pt: 0.1214 | Regression_sin_phi: 0.1196 | LR: 8.88e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3387s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0064 | max_forward: 0.7959 | max_loss: 0.4521 | max_backward: 1.8680 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 5950.3 | uss_MB: 5262.3 | child_rss_MB: 4566.8 | allocated_MB: 447.8 | reserved_MB: 27922.0 | inactive_split_MB: 1010.0 +rank7 [2026-10-28 21:17:17,194] INFO : Step 11600/50000 rank7 | Train Loss: 1.5035 | Train Loss Unweighted: 0.8050 | Train Loss Components: Auxiliary: 0.2888 | Cardinality: 0.1716 | Classification: 0.1447 | Classification_binary: 0.2597 | HitGrouping: 0.1860 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1960 | Regression_eta: 0.0026 | Regression_pt: 0.1990 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2403 | Regression_cos_phi: 0.1284 | Regression_energy: 0.0975 | Regression_eta: 0.0738 | Regression_pt: 0.0998 | Regression_sin_phi: 0.1092 | LR: 8.86e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2582s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7514 | max_loss: 0.1454 | max_backward: 1.6892 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5886.9 | uss_MB: 3729.9 | child_rss_MB: 4414.7 | allocated_MB: 449.5 | reserved_MB: 27922.0 | inactive_split_MB: 1352.4 +rank7 [2026-10-28 21:22:13,462] INFO : Step 11700/50000 rank7 | Train Loss: 1.5378 | Train Loss Unweighted: 0.8140 | Train Loss Components: Auxiliary: 0.2903 | Cardinality: 0.1812 | Classification: 0.1247 | Classification_binary: 0.2786 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2016 | Regression_eta: 0.0026 | Regression_pt: 0.2033 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2990 | Regression_cos_phi: 0.1340 | Regression_energy: 0.1099 | Regression_eta: 0.0833 | Regression_pt: 0.1134 | Regression_sin_phi: 0.1376 | LR: 8.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5935s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8039 | max_loss: 0.2576 | max_backward: 2.0737 | max_optimizer_step: 0.2033 | max_tensorboard_logging: 0.6092 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 5875.9 | uss_MB: 5262.0 | child_rss_MB: 4384.2 | allocated_MB: 451.6 | reserved_MB: 27922.0 | inactive_split_MB: 1315.7 +rank7 [2026-10-28 21:27:07,812] INFO : Step 11800/50000 rank7 | Train Loss: 1.3307 | Train Loss Unweighted: 0.7148 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1523 | Classification: 0.1135 | Classification_binary: 0.2141 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1908 | Regression_eta: 0.0025 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2437 | Regression_cos_phi: 0.1265 | Regression_energy: 0.1034 | Regression_eta: 0.0728 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0801 | LR: 8.82e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5594s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0815 | max_forward: 0.8148 | max_loss: 0.1458 | max_backward: 1.9894 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6325 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5860.3 | uss_MB: 5253.9 | child_rss_MB: 4367.1 | allocated_MB: 446.8 | reserved_MB: 27922.0 | inactive_split_MB: 1139.8 +rank7 [2026-10-28 21:31:57,941] INFO : Step 11900/50000 rank7 | Train Loss: 1.3821 | Train Loss Unweighted: 0.7440 | Train Loss Components: Auxiliary: 0.2620 | Cardinality: 0.1260 | Classification: 0.1249 | Classification_binary: 0.2250 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0016 | Regression_energy: 0.1934 | Regression_eta: 0.0026 | Regression_pt: 0.1951 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1918 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0984 | Regression_eta: 0.0558 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0882 | LR: 8.80e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9850s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0071 | max_forward: 0.7834 | max_loss: 0.2341 | max_backward: 2.0290 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 1.0813 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5861.3 | uss_MB: 5262.2 | child_rss_MB: 4377.9 | allocated_MB: 448.1 | reserved_MB: 27922.0 | inactive_split_MB: 1347.9 +rank7 [2026-10-28 21:36:37,259] INFO : Step 12000/50000 rank7 | Train Loss: 1.5318 | Train Loss Unweighted: 0.8269 | Train Loss Components: Auxiliary: 0.2953 | Cardinality: 0.1770 | Classification: 0.1326 | Classification_binary: 0.2757 | HitGrouping: 0.1863 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2042 | Regression_eta: 0.0027 | Regression_pt: 0.2087 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1163 | Classification_binary: 0.2975 | Regression_cos_phi: 0.1450 | Regression_energy: 0.1021 | Regression_eta: 0.1036 | Regression_pt: 0.1064 | Regression_sin_phi: 0.1624 | LR: 8.78e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0207s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0078 | max_forward: 0.8706 | max_loss: 0.5206 | max_backward: 2.0486 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 5858.3 | uss_MB: 5263.1 | child_rss_MB: 4331.5 | allocated_MB: 452.9 | reserved_MB: 27922.0 | inactive_split_MB: 1318.1 +rank7 [2026-10-28 21:36:38,362] INFO : Running validation on rank7 +rank7 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:36:38,363] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:37:31,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:37:31,908] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 21:38:00,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:38:01,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:38:02,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:38:03,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:38:13,470] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:38:14,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:38:14,611] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 21:38:30,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:38:33,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:38:34,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:38:35,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:38:45,271] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 21:38:45,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:38:46,116] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 21:39:01,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:39:02,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:39:03,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:39:04,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_12000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:39:14,077] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 21:44:17,716] INFO : Step 12100/50000 rank7 | Train Loss: 1.3583 | Train Loss Unweighted: 0.7218 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1448 | Classification: 0.1080 | Classification_binary: 0.2281 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1895 | Regression_eta: 0.0025 | Regression_pt: 0.1908 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0944 | Regression_eta: 0.0532 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0570 | LR: 8.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1507s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.9061 | max_loss: 0.1485 | max_backward: 1.9485 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.1474 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7031.2 | uss_MB: 6429.4 | child_rss_MB: 4320.5 | allocated_MB: 450.5 | reserved_MB: 25612.0 | inactive_split_MB: 972.7 +rank7 [2026-10-28 21:49:00,968] INFO : Step 12200/50000 rank7 | Train Loss: 1.3255 | Train Loss Unweighted: 0.7001 | Train Loss Components: Auxiliary: 0.2528 | Cardinality: 0.1423 | Classification: 0.1204 | Classification_binary: 0.1964 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1889 | Regression_eta: 0.0025 | Regression_pt: 0.1891 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2217 | Regression_cos_phi: 0.1505 | Regression_energy: 0.0902 | Regression_eta: 0.0773 | Regression_pt: 0.0916 | Regression_sin_phi: 0.1201 | LR: 8.74e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.5159s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0231 | max_forward: 0.8239 | max_loss: 0.1694 | max_backward: 1.9885 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1088 | Memory: rss_MB: 5918.8 | uss_MB: 5319.9 | child_rss_MB: 4325.8 | allocated_MB: 448.2 | reserved_MB: 27968.0 | inactive_split_MB: 892.5 +rank7 [2026-10-28 21:53:55,879] INFO : Step 12300/50000 rank7 | Train Loss: 1.5060 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2829 | Cardinality: 0.1908 | Classification: 0.1232 | Classification_binary: 0.2658 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1903 | Regression_eta: 0.0026 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2861 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1072 | Regression_eta: 0.0755 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0929 | LR: 8.72e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6871s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8651 | max_loss: 0.1513 | max_backward: 2.0574 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.5691 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1360 | Memory: rss_MB: 5920.8 | uss_MB: 5324.2 | child_rss_MB: 4312.4 | allocated_MB: 448.9 | reserved_MB: 28198.0 | inactive_split_MB: 943.2 +rank7 [2026-10-28 21:58:40,948] INFO : Step 12400/50000 rank7 | Train Loss: 1.4575 | Train Loss Unweighted: 0.7787 | Train Loss Components: Auxiliary: 0.2764 | Cardinality: 0.1678 | Classification: 0.1235 | Classification_binary: 0.2713 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1888 | Regression_eta: 0.0025 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2493 | Regression_cos_phi: 0.1040 | Regression_energy: 0.1047 | Regression_eta: 0.0733 | Regression_pt: 0.1086 | Regression_sin_phi: 0.1335 | LR: 8.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2693s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8792 | max_loss: 0.1853 | max_backward: 2.6064 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5794 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5918.6 | uss_MB: 5146.7 | child_rss_MB: 4321.3 | allocated_MB: 447.7 | reserved_MB: 29968.0 | inactive_split_MB: 1114.5 +rank7 [2026-10-28 22:03:23,385] INFO : Step 12500/50000 rank7 | Train Loss: 1.2316 | Train Loss Unweighted: 0.6535 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1365 | Classification: 0.0982 | Classification_binary: 0.1942 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0024 | Regression_pt: 0.1787 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.2069 | Regression_cos_phi: 0.1019 | Regression_energy: 0.1064 | Regression_eta: 0.0707 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0832 | LR: 8.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6717s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0067 | max_forward: 0.7170 | max_loss: 0.2498 | max_backward: 1.7709 | max_optimizer_step: 0.2052 | max_tensorboard_logging: 0.9351 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 5919.6 | uss_MB: 5320.2 | child_rss_MB: 4311.1 | allocated_MB: 447.2 | reserved_MB: 29968.0 | inactive_split_MB: 1125.8 +rank7 [2026-10-28 22:08:04,812] INFO : Step 12600/50000 rank7 | Train Loss: 1.3783 | Train Loss Unweighted: 0.7163 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1549 | Classification: 0.1166 | Classification_binary: 0.2275 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1821 | Regression_eta: 0.0028 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1960 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0741 | Regression_eta: 0.0796 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0700 | LR: 8.66e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1053s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0074 | max_forward: 0.9180 | max_loss: 0.1583 | max_backward: 2.2226 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.9629 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5922.5 | uss_MB: 3756.7 | child_rss_MB: 4318.4 | allocated_MB: 451.6 | reserved_MB: 29968.0 | inactive_split_MB: 1319.8 +rank7 [2026-10-28 22:12:51,185] INFO : Step 12700/50000 rank7 | Train Loss: 1.4203 | Train Loss Unweighted: 0.7528 | Train Loss Components: Auxiliary: 0.2746 | Cardinality: 0.1632 | Classification: 0.1236 | Classification_binary: 0.2629 | HitGrouping: 0.1806 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1047 | Regression_eta: 0.0679 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0961 | LR: 8.64e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3620s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7878 | max_loss: 0.1564 | max_backward: 1.9567 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 5919.8 | uss_MB: 5323.5 | child_rss_MB: 4328.9 | allocated_MB: 446.7 | reserved_MB: 29968.0 | inactive_split_MB: 1315.3 +rank7 [2026-10-28 22:17:38,511] INFO : Step 12800/50000 rank7 | Train Loss: 1.4257 | Train Loss Unweighted: 0.7243 | Train Loss Components: Auxiliary: 0.2730 | Cardinality: 0.1823 | Classification: 0.1101 | Classification_binary: 0.2284 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1878 | Regression_eta: 0.0028 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0907 | Regression_eta: 0.0749 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0847 | LR: 8.62e-04 | DataLoad Time: 0.0245s | Model Forward Time: 6.0947s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0188 | max_forward: 1.0001 | max_loss: 0.2153 | max_backward: 2.9413 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.6976 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 5907.7 | uss_MB: 4173.5 | child_rss_MB: 4308.3 | allocated_MB: 447.7 | reserved_MB: 30286.0 | inactive_split_MB: 1118.5 +rank7 [2026-10-28 22:22:32,702] INFO : Step 12900/50000 rank7 | Train Loss: 1.3366 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1426 | Classification: 0.1166 | Classification_binary: 0.2134 | HitGrouping: 0.1813 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1813 | Regression_eta: 0.0026 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0785 | Regression_energy: 0.1134 | Regression_eta: 0.0891 | Regression_pt: 0.1139 | Regression_sin_phi: 0.1217 | LR: 8.60e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6619s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 1.0674 | max_loss: 0.1429 | max_backward: 1.8443 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8115 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 5908.6 | uss_MB: 5318.8 | child_rss_MB: 4306.2 | allocated_MB: 449.9 | reserved_MB: 30286.0 | inactive_split_MB: 1138.6 +rank7 [2026-10-28 22:27:21,815] INFO : Step 13000/50000 rank7 | Train Loss: 1.4749 | Train Loss Unweighted: 0.7377 | Train Loss Components: Auxiliary: 0.2767 | Cardinality: 0.2032 | Classification: 0.1278 | Classification_binary: 0.2514 | HitGrouping: 0.1805 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1757 | Regression_eta: 0.0027 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2438 | Regression_cos_phi: 0.1057 | Regression_energy: 0.0871 | Regression_eta: 0.0783 | Regression_pt: 0.0912 | Regression_sin_phi: 0.1109 | LR: 8.58e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9258s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8535 | max_loss: 0.1617 | max_backward: 2.1877 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 5911.6 | uss_MB: 5323.5 | child_rss_MB: 4332.6 | allocated_MB: 450.4 | reserved_MB: 30286.0 | inactive_split_MB: 970.3 +rank7 [2026-10-28 22:27:23,211] INFO : Running validation on rank7 +rank7 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 22:27:23,211] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 22:28:27,601] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:28:28,169] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 22:28:54,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 22:28:55,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 22:28:56,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 22:28:57,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 22:29:07,182] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 22:29:08,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:29:08,605] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 22:29:24,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 22:29:25,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 22:29:26,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 22:29:27,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 22:29:37,193] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 22:29:38,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:29:38,497] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 22:29:54,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 22:29:55,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 22:29:56,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 22:29:57,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_13000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 22:30:07,115] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 22:35:15,708] INFO : Step 13100/50000 rank7 | Train Loss: 1.3479 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1865 | Classification: 0.0950 | Classification_binary: 0.2288 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1762 | Regression_eta: 0.0022 | Regression_pt: 0.1780 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.2505 | Regression_cos_phi: 0.0894 | Regression_energy: 0.1066 | Regression_eta: 0.0703 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0843 | LR: 8.56e-04 | DataLoad Time: 0.0220s | Model Forward Time: 6.4840s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0856 | max_forward: 1.8916 | max_loss: 0.7517 | max_backward: 2.8337 | max_optimizer_step: 0.2215 | max_tensorboard_logging: 1.0483 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7077.4 | uss_MB: 6490.5 | child_rss_MB: 4325.4 | allocated_MB: 449.6 | reserved_MB: 27770.0 | inactive_split_MB: 1139.8 +rank7 [2026-10-28 22:40:04,074] INFO : Step 13200/50000 rank7 | Train Loss: 1.2817 | Train Loss Unweighted: 0.6446 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1780 | Classification: 0.1014 | Classification_binary: 0.1920 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1727 | Regression_eta: 0.0021 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0885 | Regression_eta: 0.0511 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0513 | LR: 8.54e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6865s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0061 | max_forward: 0.7952 | max_loss: 0.1428 | max_backward: 1.9281 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7515 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6000.4 | uss_MB: 5415.4 | child_rss_MB: 4302.0 | allocated_MB: 446.4 | reserved_MB: 29822.0 | inactive_split_MB: 1319.7 +rank7 [2026-10-28 22:44:58,539] INFO : Step 13300/50000 rank7 | Train Loss: 1.3968 | Train Loss Unweighted: 0.6989 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1964 | Classification: 0.1115 | Classification_binary: 0.2401 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1705 | Regression_eta: 0.0023 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.2406 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0947 | Regression_eta: 0.0594 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0554 | LR: 8.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.1100 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.2568 | max_backward: 2.0103 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.5869 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6000.4 | uss_MB: 5419.8 | child_rss_MB: 4304.4 | allocated_MB: 450.6 | reserved_MB: 29826.0 | inactive_split_MB: 1137.8 +rank7 [2026-10-28 22:49:50,192] INFO : Step 13400/50000 rank7 | Train Loss: 2.1571 | Train Loss Unweighted: 0.9627 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.5171 | Classification: 0.1578 | Classification_binary: 0.3195 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2389 | Regression_eta: 0.0035 | Regression_pt: 0.2397 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.3140 | Regression_cos_phi: 0.1422 | Regression_energy: 0.1181 | Regression_eta: 0.1028 | Regression_pt: 0.1224 | Regression_sin_phi: 0.1348 | LR: 8.50e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.6024s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.8565 | max_loss: 0.1469 | max_backward: 3.0149 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.6025 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5979.0 | uss_MB: 5416.7 | child_rss_MB: 4321.2 | allocated_MB: 451.0 | reserved_MB: 29828.0 | inactive_split_MB: 1055.8 +rank7 [2026-10-28 22:54:39,510] INFO : Step 13500/50000 rank7 | Train Loss: 1.2646 | Train Loss Unweighted: 0.6738 | Train Loss Components: Auxiliary: 0.2488 | Cardinality: 0.1535 | Classification: 0.0940 | Classification_binary: 0.1997 | HitGrouping: 0.1847 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1875 | Regression_eta: 0.0019 | Regression_pt: 0.1884 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0956 | Regression_eta: 0.0471 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0886 | LR: 8.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4625s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7684 | max_loss: 0.1466 | max_backward: 1.8396 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.6131 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1368 | Memory: rss_MB: 5981.3 | uss_MB: 5418.2 | child_rss_MB: 4328.6 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1332.2 +rank7 [2026-10-28 22:59:25,131] INFO : Step 13600/50000 rank7 | Train Loss: 1.2808 | Train Loss Unweighted: 0.6516 | Train Loss Components: Auxiliary: 0.2438 | Cardinality: 0.1721 | Classification: 0.1008 | Classification_binary: 0.2123 | HitGrouping: 0.1737 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1678 | Regression_eta: 0.0020 | Regression_pt: 0.1662 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.2322 | Regression_cos_phi: 0.1028 | Regression_energy: 0.1133 | Regression_eta: 0.0720 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0879 | LR: 8.45e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8505s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7847 | max_loss: 0.1460 | max_backward: 2.0245 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.9384 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5969.9 | uss_MB: 5407.7 | child_rss_MB: 4335.7 | allocated_MB: 449.5 | reserved_MB: 29828.0 | inactive_split_MB: 932.7 +rank7 [2026-10-28 23:04:05,987] INFO : Step 13700/50000 rank7 | Train Loss: 1.4912 | Train Loss Unweighted: 0.7665 | Train Loss Components: Auxiliary: 0.2905 | Cardinality: 0.2254 | Classification: 0.1233 | Classification_binary: 0.2695 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1832 | Regression_eta: 0.0022 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0907 | Regression_eta: 0.0584 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0751 | LR: 8.43e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7309s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0076 | max_forward: 0.8411 | max_loss: 0.1547 | max_backward: 1.7834 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 5977.7 | uss_MB: 5245.5 | child_rss_MB: 4328.4 | allocated_MB: 452.0 | reserved_MB: 29828.0 | inactive_split_MB: 1354.3 +rank7 [2026-10-28 23:09:16,294] INFO : Step 13800/50000 rank7 | Train Loss: 1.3073 | Train Loss Unweighted: 0.6864 | Train Loss Components: Auxiliary: 0.2532 | Cardinality: 0.1632 | Classification: 0.1022 | Classification_binary: 0.2094 | HitGrouping: 0.1854 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1851 | Regression_eta: 0.0020 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0947 | Regression_eta: 0.0541 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0842 | LR: 8.41e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4796s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7604 | max_loss: 0.2459 | max_backward: 1.9067 | max_optimizer_step: 0.2968 | max_tensorboard_logging: 0.5877 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 5975.1 | uss_MB: 5417.1 | child_rss_MB: 4324.1 | allocated_MB: 447.6 | reserved_MB: 29828.0 | inactive_split_MB: 1210.0 +rank7 [2026-10-28 23:14:10,152] INFO : Step 13900/50000 rank7 | Train Loss: 1.5530 | Train Loss Unweighted: 0.8691 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.1914 | Classification: 0.1874 | Classification_binary: 0.2223 | HitGrouping: 0.2003 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2268 | Regression_eta: 0.0023 | Regression_pt: 0.2273 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1440 | Classification_binary: 0.2629 | Regression_cos_phi: 0.2328 | Regression_energy: 0.1360 | Regression_eta: 0.0985 | Regression_pt: 0.1424 | Regression_sin_phi: 0.1750 | LR: 8.39e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2880s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7479 | max_loss: 0.1425 | max_backward: 1.6678 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 5972.0 | uss_MB: 5412.6 | child_rss_MB: 4308.7 | allocated_MB: 448.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.5 +rank7 [2026-10-28 23:18:59,721] INFO : Step 14000/50000 rank7 | Train Loss: 1.5016 | Train Loss Unweighted: 0.7565 | Train Loss Components: Auxiliary: 0.2922 | Cardinality: 0.2233 | Classification: 0.1350 | Classification_binary: 0.2521 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1815 | Regression_eta: 0.0023 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0896 | Classification_binary: 0.2564 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0908 | Regression_eta: 0.0615 | Regression_pt: 0.0939 | Regression_sin_phi: 0.1396 | LR: 8.37e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1757s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.8543 | max_loss: 0.1567 | max_backward: 2.1957 | max_optimizer_step: 0.2093 | max_tensorboard_logging: 1.0107 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 5978.8 | uss_MB: 5421.0 | child_rss_MB: 4315.0 | allocated_MB: 450.7 | reserved_MB: 29828.0 | inactive_split_MB: 1144.8 +rank7 [2026-10-28 23:19:00,860] INFO : Running validation on rank7 +rank7 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:19:00,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 23:19:52,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:19:53,516] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-28 23:20:19,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 23:20:20,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 23:20:21,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 23:20:22,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 23:20:32,472] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:20:33,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:20:33,745] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-28 23:20:49,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 23:20:50,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 23:20:51,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 23:20:52,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 23:21:02,273] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:21:03,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:21:04,038] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-28 23:21:19,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 23:21:20,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 23:21:20,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 23:21:21,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_14000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 23:21:31,619] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 23:26:43,451] INFO : Step 14100/50000 rank7 | Train Loss: 1.3083 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2508 | Cardinality: 0.1622 | Classification: 0.1004 | Classification_binary: 0.2286 | HitGrouping: 0.1761 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1610 | Regression_eta: 0.0029 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0865 | Regression_eta: 0.0810 | Regression_pt: 0.0896 | Regression_sin_phi: 0.1035 | LR: 8.35e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.5299s | Phase Times: max_pre_step_barrier: 0.0097 | max_batch_to_device: 0.0203 | max_forward: 1.1303 | max_loss: 0.4972 | max_backward: 2.1605 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 1.1828 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0645 | Memory: rss_MB: 6972.5 | uss_MB: 6419.7 | child_rss_MB: 4332.8 | allocated_MB: 448.5 | reserved_MB: 28540.0 | inactive_split_MB: 1328.4 +rank7 [2026-10-28 23:31:31,720] INFO : Step 14200/50000 rank7 | Train Loss: 1.1208 | Train Loss Unweighted: 0.6043 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1208 | Classification: 0.0910 | Classification_binary: 0.1748 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1667 | Regression_eta: 0.0019 | Regression_pt: 0.1680 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0883 | Regression_eta: 0.0868 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0726 | LR: 8.32e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6164s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 0.7373 | max_loss: 0.1872 | max_backward: 1.8445 | max_optimizer_step: 0.3052 | max_tensorboard_logging: 0.7694 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6028.2 | uss_MB: 5346.7 | child_rss_MB: 4328.5 | allocated_MB: 445.0 | reserved_MB: 28540.0 | inactive_split_MB: 1303.6 +rank7 [2026-10-28 23:36:16,487] INFO : Step 14300/50000 rank7 | Train Loss: 1.2335 | Train Loss Unweighted: 0.6435 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1341 | Classification: 0.1039 | Classification_binary: 0.2166 | HitGrouping: 0.1760 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1582 | Regression_eta: 0.0020 | Regression_pt: 0.1603 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0766 | Regression_eta: 0.0488 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0691 | LR: 8.30e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6738s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0074 | max_forward: 0.7735 | max_loss: 0.1491 | max_backward: 1.8495 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.8378 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 6024.2 | uss_MB: 5490.6 | child_rss_MB: 4322.4 | allocated_MB: 448.7 | reserved_MB: 30540.0 | inactive_split_MB: 1159.7 +rank7 [2026-10-28 23:40:58,337] INFO : Step 14400/50000 rank7 | Train Loss: 1.3659 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1757 | Classification: 0.1031 | Classification_binary: 0.2605 | HitGrouping: 0.1778 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1634 | Regression_eta: 0.0023 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2562 | Regression_cos_phi: 0.1153 | Regression_energy: 0.0808 | Regression_eta: 0.0768 | Regression_pt: 0.0849 | Regression_sin_phi: 0.1079 | LR: 8.28e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7768s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0073 | max_forward: 0.8101 | max_loss: 0.5082 | max_backward: 1.9309 | max_optimizer_step: 0.2048 | max_tensorboard_logging: 0.7999 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6003.4 | uss_MB: 5490.5 | child_rss_MB: 4327.7 | allocated_MB: 450.7 | reserved_MB: 30540.0 | inactive_split_MB: 1305.7 +rank7 [2026-10-28 23:45:42,941] INFO : Step 14500/50000 rank7 | Train Loss: 1.1290 | Train Loss Unweighted: 0.5841 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1339 | Classification: 0.0915 | Classification_binary: 0.1811 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1528 | Regression_eta: 0.0020 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1920 | Regression_cos_phi: 0.1250 | Regression_energy: 0.0907 | Regression_eta: 0.0844 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0916 | LR: 8.26e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.6344s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7414 | max_loss: 0.2506 | max_backward: 1.9008 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8751 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6000.9 | uss_MB: 5491.1 | child_rss_MB: 4333.1 | allocated_MB: 444.3 | reserved_MB: 30540.0 | inactive_split_MB: 1327.1 +rank7 [2026-10-28 23:50:27,494] INFO : Step 14600/50000 rank7 | Train Loss: 1.1887 | Train Loss Unweighted: 0.6121 | Train Loss Components: Auxiliary: 0.2293 | Cardinality: 0.1441 | Classification: 0.0965 | Classification_binary: 0.1973 | HitGrouping: 0.1725 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1566 | Regression_eta: 0.0020 | Regression_pt: 0.1575 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0794 | Regression_eta: 0.0497 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0839 | LR: 8.24e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4481s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 0.7735 | max_loss: 0.1517 | max_backward: 1.9283 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.6314 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 5992.8 | uss_MB: 5481.7 | child_rss_MB: 4303.1 | allocated_MB: 445.6 | reserved_MB: 32632.0 | inactive_split_MB: 1135.7 +rank7 [2026-10-28 23:55:12,843] INFO : Step 14700/50000 rank7 | Train Loss: 1.3698 | Train Loss Unweighted: 0.7149 | Train Loss Components: Auxiliary: 0.2682 | Cardinality: 0.1672 | Classification: 0.1112 | Classification_binary: 0.2640 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1671 | Regression_eta: 0.0023 | Regression_pt: 0.1679 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2401 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0989 | Regression_eta: 0.0729 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0976 | LR: 8.21e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6455s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0067 | max_forward: 0.9622 | max_loss: 0.1844 | max_backward: 2.1217 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4976 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 5998.3 | uss_MB: 5491.0 | child_rss_MB: 4324.9 | allocated_MB: 449.0 | reserved_MB: 32632.0 | inactive_split_MB: 1349.9 +rank7 [2026-10-28 23:59:56,485] INFO : Step 14800/50000 rank7 | Train Loss: 1.1852 | Train Loss Unweighted: 0.6196 | Train Loss Components: Auxiliary: 0.2281 | Cardinality: 0.1316 | Classification: 0.0864 | Classification_binary: 0.1926 | HitGrouping: 0.1741 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1675 | Regression_eta: 0.0021 | Regression_pt: 0.1687 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2088 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0912 | Regression_eta: 0.0633 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0612 | LR: 8.19e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.7609s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0210 | max_forward: 1.6831 | max_loss: 0.3323 | max_backward: 2.9261 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 5996.7 | uss_MB: 5491.3 | child_rss_MB: 4327.9 | allocated_MB: 448.6 | reserved_MB: 32632.0 | inactive_split_MB: 1152.5 +rank7 [2026-10-29 00:04:46,438] INFO : Step 14900/50000 rank7 | Train Loss: 1.0779 | Train Loss Unweighted: 0.5773 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1125 | Classification: 0.0890 | Classification_binary: 0.1668 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1572 | Regression_eta: 0.0017 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0811 | Regression_eta: 0.0449 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0628 | LR: 8.17e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1958s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7794 | max_loss: 0.1574 | max_backward: 1.6766 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0080 | Memory: rss_MB: 5991.4 | uss_MB: 3760.0 | child_rss_MB: 4302.8 | allocated_MB: 444.3 | reserved_MB: 32632.0 | inactive_split_MB: 1111.7 +rank7 [2026-10-29 00:09:43,124] INFO : Step 15000/50000 rank7 | Train Loss: 1.3450 | Train Loss Unweighted: 0.6824 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1707 | Classification: 0.1003 | Classification_binary: 0.2364 | HitGrouping: 0.1787 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1700 | Regression_eta: 0.0025 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0805 | Regression_eta: 0.0585 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0862 | LR: 8.15e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7502s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 1.1550 | max_loss: 0.2524 | max_backward: 2.0533 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0412 | Memory: rss_MB: 5994.4 | uss_MB: 5492.7 | child_rss_MB: 4321.2 | allocated_MB: 447.7 | reserved_MB: 32632.0 | inactive_split_MB: 1336.9 +rank7 [2026-10-29 00:09:44,171] INFO : Running validation on rank7 +rank7 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 00:09:44,171] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:10:38,761] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:10:39,318] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 00:11:05,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:11:07,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:11:08,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:11:09,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:11:19,557] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 00:11:20,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:11:20,773] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 00:11:36,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:11:37,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:11:38,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:11:39,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:11:49,669] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 00:11:50,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:11:50,985] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 00:12:07,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:12:07,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:12:08,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:12:09,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_15000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:12:19,534] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 00:17:30,254] INFO : Step 15100/50000 rank7 | Train Loss: 1.2437 | Train Loss Unweighted: 0.6630 | Train Loss Components: Auxiliary: 0.2435 | Cardinality: 0.1215 | Classification: 0.1020 | Classification_binary: 0.2266 | HitGrouping: 0.1727 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1624 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2433 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0724 | Regression_eta: 0.0698 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0828 | LR: 8.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3981s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8767 | max_loss: 0.2709 | max_backward: 2.4594 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.8856 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6985.5 | uss_MB: 6175.3 | child_rss_MB: 4330.0 | allocated_MB: 448.2 | reserved_MB: 26960.0 | inactive_split_MB: 1312.8 +rank7 [2026-10-29 00:22:16,917] INFO : Step 15200/50000 rank7 | Train Loss: 1.1992 | Train Loss Unweighted: 0.6195 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1407 | Classification: 0.0995 | Classification_binary: 0.1753 | HitGrouping: 0.1764 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1687 | Regression_eta: 0.0020 | Regression_pt: 0.1716 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0754 | Regression_eta: 0.0609 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0973 | LR: 8.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2234s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7769 | max_loss: 0.1628 | max_backward: 2.3723 | max_optimizer_step: 0.1824 | max_tensorboard_logging: 0.6383 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6037.1 | uss_MB: 5217.8 | child_rss_MB: 4332.9 | allocated_MB: 446.8 | reserved_MB: 29066.0 | inactive_split_MB: 1208.0 +rank7 [2026-10-29 00:27:05,718] INFO : Step 15300/50000 rank7 | Train Loss: 1.3203 | Train Loss Unweighted: 0.6652 | Train Loss Components: Auxiliary: 0.2560 | Cardinality: 0.1771 | Classification: 0.1026 | Classification_binary: 0.2124 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1711 | Regression_eta: 0.0020 | Regression_pt: 0.1745 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2142 | Regression_cos_phi: 0.1607 | Regression_energy: 0.0970 | Regression_eta: 0.0618 | Regression_pt: 0.0992 | Regression_sin_phi: 0.1353 | LR: 8.08e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3159s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0409 | max_forward: 0.8277 | max_loss: 0.1530 | max_backward: 1.8477 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5198 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6038.4 | uss_MB: 5532.1 | child_rss_MB: 4330.7 | allocated_MB: 449.7 | reserved_MB: 29066.0 | inactive_split_MB: 1228.7 +rank7 [2026-10-29 00:31:57,726] INFO : Step 15400/50000 rank7 | Train Loss: 1.4067 | Train Loss Unweighted: 0.6985 | Train Loss Components: Auxiliary: 0.2762 | Cardinality: 0.1986 | Classification: 0.1129 | Classification_binary: 0.2508 | HitGrouping: 0.1754 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1635 | Regression_eta: 0.0021 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2730 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0754 | Regression_eta: 0.0567 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0885 | LR: 8.05e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6642s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.8017 | max_loss: 0.2558 | max_backward: 1.9458 | max_optimizer_step: 0.2648 | max_tensorboard_logging: 0.7016 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6025.6 | uss_MB: 5522.1 | child_rss_MB: 4326.9 | allocated_MB: 450.6 | reserved_MB: 30946.0 | inactive_split_MB: 1020.4 +rank7 [2026-10-29 00:36:52,789] INFO : Step 15500/50000 rank7 | Train Loss: 1.1702 | Train Loss Unweighted: 0.6052 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1424 | Classification: 0.0876 | Classification_binary: 0.1938 | HitGrouping: 0.1777 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1593 | Regression_eta: 0.0018 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0988 | Regression_eta: 0.0435 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0859 | LR: 8.03e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4206s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 1.0922 | max_loss: 0.1930 | max_backward: 1.8200 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6032.6 | uss_MB: 5529.6 | child_rss_MB: 4349.5 | allocated_MB: 448.1 | reserved_MB: 30946.0 | inactive_split_MB: 1344.9 +rank7 [2026-10-29 00:41:49,267] INFO : Step 15600/50000 rank7 | Train Loss: 1.0865 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1193 | Classification: 0.0923 | Classification_binary: 0.1726 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1478 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0836 | Regression_eta: 0.0367 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0725 | LR: 8.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9280s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.8649 | max_loss: 0.1396 | max_backward: 2.1241 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6027.5 | uss_MB: 5523.6 | child_rss_MB: 4345.1 | allocated_MB: 447.2 | reserved_MB: 31066.0 | inactive_split_MB: 1218.1 +rank7 [2026-10-29 00:46:43,871] INFO : Step 15700/50000 rank7 | Train Loss: 1.2991 | Train Loss Unweighted: 0.6661 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1611 | Classification: 0.1098 | Classification_binary: 0.2313 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1600 | Regression_eta: 0.0019 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0705 | Regression_eta: 0.0399 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0815 | LR: 7.98e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7551s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.8297 | max_loss: 0.4957 | max_backward: 1.9048 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.9028 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6033.1 | uss_MB: 5526.7 | child_rss_MB: 4341.4 | allocated_MB: 452.6 | reserved_MB: 31066.0 | inactive_split_MB: 1340.8 +rank7 [2026-10-29 00:51:29,696] INFO : Step 15800/50000 rank7 | Train Loss: 1.3026 | Train Loss Unweighted: 0.6388 | Train Loss Components: Auxiliary: 0.2505 | Cardinality: 0.1741 | Classification: 0.0946 | Classification_binary: 0.2235 | HitGrouping: 0.1758 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1571 | Regression_eta: 0.0023 | Regression_pt: 0.1587 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.2296 | Regression_cos_phi: 0.1159 | Regression_energy: 0.0724 | Regression_eta: 0.0592 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0918 | LR: 7.96e-04 | DataLoad Time: 0.0236s | Model Forward Time: 5.0246s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.9309 | max_loss: 0.1815 | max_backward: 2.2856 | max_optimizer_step: 0.2820 | max_tensorboard_logging: 0.6634 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0447 | Memory: rss_MB: 6034.1 | uss_MB: 5534.7 | child_rss_MB: 4311.8 | allocated_MB: 446.5 | reserved_MB: 31066.0 | inactive_split_MB: 1350.8 +rank7 [2026-10-29 00:56:25,007] INFO : Step 15900/50000 rank7 | Train Loss: 1.1043 | Train Loss Unweighted: 0.5416 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1237 | Classification: 0.0843 | Classification_binary: 0.1732 | HitGrouping: 0.1745 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0020 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0528 | Regression_eta: 0.0509 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0541 | LR: 7.94e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1837s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.7524 | max_loss: 0.1447 | max_backward: 1.8070 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6032.0 | uss_MB: 5531.5 | child_rss_MB: 4316.5 | allocated_MB: 445.6 | reserved_MB: 31066.0 | inactive_split_MB: 1396.5 +rank7 [2026-10-29 01:01:16,824] INFO : Step 16000/50000 rank7 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1630 | Classification: 0.1095 | Classification_binary: 0.2196 | HitGrouping: 0.1734 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1477 | Regression_eta: 0.0021 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2349 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0736 | Regression_eta: 0.0829 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0964 | LR: 7.91e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.3010s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.1072 | max_forward: 1.1717 | max_loss: 0.5288 | max_backward: 2.4568 | max_optimizer_step: 0.2080 | max_tensorboard_logging: 0.5190 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6022.2 | uss_MB: 5533.7 | child_rss_MB: 4264.2 | allocated_MB: 449.6 | reserved_MB: 31066.0 | inactive_split_MB: 1159.9 +rank7 [2026-10-29 01:01:17,883] INFO : Running validation on rank7 +rank7 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 01:01:17,883] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 01:02:10,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:02:11,392] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 01:02:39,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 01:02:40,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 01:02:42,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 01:02:44,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 01:02:54,729] INFO : Time taken to make predictions on device 7 is: 0.29 min +rank7 [2026-10-29 01:02:55,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:02:55,636] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 01:03:11,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 01:03:12,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 01:03:13,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 01:03:14,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 01:03:24,927] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:03:25,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:03:26,084] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 01:03:41,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 01:03:42,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 01:03:43,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 01:03:43,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_16000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 01:03:53,975] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 01:09:14,175] INFO : Step 16100/50000 rank7 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6260 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1715 | Classification: 0.0996 | Classification_binary: 0.2338 | HitGrouping: 0.1717 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1437 | Regression_eta: 0.0025 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2501 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0734 | Regression_eta: 0.0597 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0836 | LR: 7.89e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0657s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0474 | max_forward: 0.8850 | max_loss: 0.1555 | max_backward: 1.9221 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0166 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6914.9 | uss_MB: 6418.8 | child_rss_MB: 4280.7 | allocated_MB: 448.6 | reserved_MB: 27110.0 | inactive_split_MB: 1182.9 +rank7 [2026-10-29 01:14:08,171] INFO : Step 16200/50000 rank7 | Train Loss: 1.1530 | Train Loss Unweighted: 0.5851 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1389 | Classification: 0.0886 | Classification_binary: 0.1817 | HitGrouping: 0.1766 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1557 | Regression_eta: 0.0019 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0764 | Regression_energy: 0.1051 | Regression_eta: 0.0382 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0726 | LR: 7.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2106s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7360 | max_loss: 0.1626 | max_backward: 1.8267 | max_optimizer_step: 0.2116 | max_tensorboard_logging: 0.5135 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6102.3 | uss_MB: 5607.7 | child_rss_MB: 4283.8 | allocated_MB: 446.4 | reserved_MB: 29090.0 | inactive_split_MB: 1367.7 +rank7 [2026-10-29 01:19:03,891] INFO : Step 16300/50000 rank7 | Train Loss: 1.2092 | Train Loss Unweighted: 0.6300 | Train Loss Components: Auxiliary: 0.2365 | Cardinality: 0.1262 | Classification: 0.0998 | Classification_binary: 0.2051 | HitGrouping: 0.1752 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1601 | Regression_eta: 0.0023 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1941 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0847 | Regression_eta: 0.0624 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0945 | LR: 7.84e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2087s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0141 | max_forward: 1.0691 | max_loss: 0.2666 | max_backward: 2.3705 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.5575 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6102.1 | uss_MB: 5609.3 | child_rss_MB: 4283.6 | allocated_MB: 446.8 | reserved_MB: 29090.0 | inactive_split_MB: 1005.4 +rank7 [2026-10-29 01:23:56,183] INFO : Step 16400/50000 rank7 | Train Loss: 1.2153 | Train Loss Unweighted: 0.6126 | Train Loss Components: Auxiliary: 0.2401 | Cardinality: 0.1511 | Classification: 0.0935 | Classification_binary: 0.2331 | HitGrouping: 0.1702 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0647 | Regression_eta: 0.0500 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0766 | LR: 7.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6844s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.8340 | max_loss: 0.1504 | max_backward: 1.9918 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.6293 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0733 | Memory: rss_MB: 6072.5 | uss_MB: 5599.0 | child_rss_MB: 4227.8 | allocated_MB: 448.2 | reserved_MB: 29090.0 | inactive_split_MB: 1189.9 +rank7 [2026-10-29 01:28:51,667] INFO : Step 16500/50000 rank7 | Train Loss: 1.1717 | Train Loss Unweighted: 0.5990 | Train Loss Components: Auxiliary: 0.2285 | Cardinality: 0.1444 | Classification: 0.0879 | Classification_binary: 0.1925 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1569 | Regression_eta: 0.0018 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0808 | Regression_eta: 0.0494 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0760 | LR: 7.79e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.3067s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.7506 | max_loss: 0.1486 | max_backward: 1.8713 | max_optimizer_step: 0.2078 | max_tensorboard_logging: 0.5058 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6067.8 | uss_MB: 5607.6 | child_rss_MB: 4166.7 | allocated_MB: 447.4 | reserved_MB: 31126.0 | inactive_split_MB: 949.3 +rank7 [2026-10-29 01:33:46,138] INFO : Step 16600/50000 rank7 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1608 | Classification: 0.0952 | Classification_binary: 0.1687 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1464 | Regression_eta: 0.0019 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0973 | Regression_energy: 0.0698 | Regression_eta: 0.0602 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0833 | LR: 7.77e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1718s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8772 | max_loss: 0.1827 | max_backward: 2.2759 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.9112 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0407 | Memory: rss_MB: 6070.2 | uss_MB: 5610.8 | child_rss_MB: 4160.5 | allocated_MB: 448.2 | reserved_MB: 31126.0 | inactive_split_MB: 1365.7 +rank7 [2026-10-29 01:38:38,619] INFO : Step 16700/50000 rank7 | Train Loss: 1.3901 | Train Loss Unweighted: 0.6846 | Train Loss Components: Auxiliary: 0.2647 | Cardinality: 0.2076 | Classification: 0.1118 | Classification_binary: 0.2314 | HitGrouping: 0.1713 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1679 | Regression_eta: 0.0028 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2396 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0857 | Regression_eta: 0.0865 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0984 | LR: 7.74e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.9121s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0070 | max_forward: 0.8568 | max_loss: 0.1660 | max_backward: 2.0415 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.8464 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6068.6 | uss_MB: 5609.6 | child_rss_MB: 4155.7 | allocated_MB: 448.5 | reserved_MB: 33178.0 | inactive_split_MB: 1340.9 +rank7 [2026-10-29 01:43:30,909] INFO : Step 16800/50000 rank7 | Train Loss: 1.2916 | Train Loss Unweighted: 0.6586 | Train Loss Components: Auxiliary: 0.2564 | Cardinality: 0.1567 | Classification: 0.1048 | Classification_binary: 0.2239 | HitGrouping: 0.1774 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1626 | Regression_eta: 0.0023 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0640 | Regression_eta: 0.0621 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0816 | LR: 7.72e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.6004s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 1.0578 | max_loss: 0.2337 | max_backward: 3.7676 | max_optimizer_step: 0.2191 | max_tensorboard_logging: 0.7256 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6068.0 | uss_MB: 5014.8 | child_rss_MB: 4160.2 | allocated_MB: 449.7 | reserved_MB: 33178.0 | inactive_split_MB: 1309.6 +rank7 [2026-10-29 01:48:23,688] INFO : Step 16900/50000 rank7 | Train Loss: 1.0618 | Train Loss Unweighted: 0.5433 | Train Loss Components: Auxiliary: 0.2072 | Cardinality: 0.1309 | Classification: 0.0836 | Classification_binary: 0.1557 | HitGrouping: 0.1729 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0761 | Regression_eta: 0.0492 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0774 | LR: 7.69e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3827s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7342 | max_loss: 0.1405 | max_backward: 1.7751 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7288 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 6066.4 | uss_MB: 5606.5 | child_rss_MB: 4155.5 | allocated_MB: 447.5 | reserved_MB: 33178.0 | inactive_split_MB: 1332.5 +rank7 [2026-10-29 01:53:19,525] INFO : Step 17000/50000 rank7 | Train Loss: 1.2176 | Train Loss Unweighted: 0.6165 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1228 | Classification: 0.0975 | Classification_binary: 0.2108 | HitGrouping: 0.1722 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1508 | Regression_eta: 0.0023 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.2222 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0692 | Regression_eta: 0.0649 | Regression_pt: 0.0730 | Regression_sin_phi: 0.1042 | LR: 7.67e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.6713s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0759 | max_forward: 0.8405 | max_loss: 0.1467 | max_backward: 2.1851 | max_optimizer_step: 0.2846 | max_tensorboard_logging: 0.6230 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0647 | Memory: rss_MB: 6066.8 | uss_MB: 5325.7 | child_rss_MB: 4160.9 | allocated_MB: 447.3 | reserved_MB: 33178.0 | inactive_split_MB: 1187.9 +rank7 [2026-10-29 01:53:20,625] INFO : Running validation on rank7 +rank7 [2026-10-29 01:53:20,625] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 01:53:20,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 01:54:26,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:54:27,392] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 01:55:04,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 01:55:05,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 01:55:06,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 01:55:07,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 01:55:17,922] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:55:18,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:55:18,855] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 01:55:34,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 01:55:35,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 01:55:36,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 01:55:38,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 01:55:48,045] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:55:49,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:55:50,414] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 01:56:05,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 01:56:06,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 01:56:07,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 01:56:07,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_17000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 01:56:17,959] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 02:01:32,243] INFO : Step 17100/50000 rank7 | Train Loss: 1.2137 | Train Loss Unweighted: 0.6111 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1646 | Classification: 0.0937 | Classification_binary: 0.2254 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1432 | Regression_eta: 0.0018 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0706 | Regression_eta: 0.0386 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0696 | LR: 7.64e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1159s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8536 | max_loss: 0.1516 | max_backward: 3.2476 | max_optimizer_step: 0.2165 | max_tensorboard_logging: 0.7950 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1062 | Memory: rss_MB: 7214.1 | uss_MB: 6749.1 | child_rss_MB: 4165.8 | allocated_MB: 451.2 | reserved_MB: 27986.0 | inactive_split_MB: 1175.4 +rank7 [2026-10-29 02:06:24,501] INFO : Step 17200/50000 rank7 | Train Loss: 1.0955 | Train Loss Unweighted: 0.5490 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1299 | Classification: 0.0922 | Classification_binary: 0.1802 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1362 | Regression_eta: 0.0018 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0689 | Regression_eta: 0.0460 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0601 | LR: 7.62e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.6203s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0483 | max_forward: 0.8616 | max_loss: 0.1420 | max_backward: 1.8929 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6907 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0719 | Memory: rss_MB: 6135.0 | uss_MB: 4409.2 | child_rss_MB: 4163.5 | allocated_MB: 448.1 | reserved_MB: 29950.0 | inactive_split_MB: 1185.7 +rank7 [2026-10-29 02:11:06,510] INFO : Step 17300/50000 rank7 | Train Loss: 1.1224 | Train Loss Unweighted: 0.5522 | Train Loss Components: Auxiliary: 0.2218 | Cardinality: 0.1419 | Classification: 0.0853 | Classification_binary: 0.1809 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1401 | Regression_eta: 0.0019 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0790 | Regression_eta: 0.0543 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0688 | LR: 7.59e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7527s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0068 | max_forward: 0.8291 | max_loss: 0.2549 | max_backward: 2.1651 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5744 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 6144.2 | uss_MB: 5679.0 | child_rss_MB: 4169.9 | allocated_MB: 450.1 | reserved_MB: 29950.0 | inactive_split_MB: 1430.9 +rank7 [2026-10-29 02:16:07,514] INFO : Step 17400/50000 rank7 | Train Loss: 1.2777 | Train Loss Unweighted: 0.6187 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1748 | Classification: 0.0887 | Classification_binary: 0.2323 | HitGrouping: 0.1735 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1457 | Regression_eta: 0.0020 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0820 | Regression_eta: 0.0622 | Regression_pt: 0.0848 | Regression_sin_phi: 0.1088 | LR: 7.57e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.5480s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0422 | max_forward: 0.8832 | max_loss: 0.5004 | max_backward: 1.9060 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6136.4 | uss_MB: 5671.3 | child_rss_MB: 4167.5 | allocated_MB: 454.2 | reserved_MB: 29956.0 | inactive_split_MB: 1394.9 +rank7 [2026-10-29 02:21:02,096] INFO : Step 17500/50000 rank7 | Train Loss: 1.1214 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2278 | Cardinality: 0.1341 | Classification: 0.0869 | Classification_binary: 0.1874 | HitGrouping: 0.1683 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1550 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0721 | Regression_eta: 0.0480 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0531 | LR: 7.54e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2405s | Phase Times: max_pre_step_barrier: 0.0096 | max_batch_to_device: 0.0170 | max_forward: 1.4549 | max_loss: 0.3479 | max_backward: 2.3063 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5754 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6143.3 | uss_MB: 5679.6 | child_rss_MB: 4161.3 | allocated_MB: 447.2 | reserved_MB: 31972.0 | inactive_split_MB: 1426.1 +rank7 [2026-10-29 02:25:59,308] INFO : Step 17600/50000 rank7 | Train Loss: 1.0652 | Train Loss Unweighted: 0.5259 | Train Loss Components: Auxiliary: 0.2050 | Cardinality: 0.1133 | Classification: 0.0850 | Classification_binary: 0.1628 | HitGrouping: 0.1708 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1364 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0671 | Regression_eta: 0.0486 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0699 | LR: 7.52e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.4700s | Phase Times: max_pre_step_barrier: 0.1079 | max_batch_to_device: 0.0068 | max_forward: 0.7389 | max_loss: 0.2452 | max_backward: 1.7699 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7048 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6146.6 | uss_MB: 5679.4 | child_rss_MB: 4171.6 | allocated_MB: 449.1 | reserved_MB: 31974.0 | inactive_split_MB: 1386.9 +rank7 [2026-10-29 02:30:47,051] INFO : Step 17700/50000 rank7 | Train Loss: 1.3036 | Train Loss Unweighted: 0.6356 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.2165 | Classification: 0.1003 | Classification_binary: 0.2046 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1620 | Regression_eta: 0.0022 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0819 | Regression_eta: 0.0775 | Regression_pt: 0.0858 | Regression_sin_phi: 0.1059 | LR: 7.49e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7700s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0333 | max_forward: 0.8093 | max_loss: 0.1520 | max_backward: 1.9377 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.8622 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6146.5 | uss_MB: 5682.4 | child_rss_MB: 4172.2 | allocated_MB: 446.5 | reserved_MB: 31974.0 | inactive_split_MB: 1189.6 +rank7 [2026-10-29 02:35:37,695] INFO : Step 17800/50000 rank7 | Train Loss: 1.1703 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2306 | Cardinality: 0.1502 | Classification: 0.0933 | Classification_binary: 0.2058 | HitGrouping: 0.1616 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1439 | Regression_eta: 0.0020 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2070 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0728 | Regression_eta: 0.0610 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0779 | LR: 7.47e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0215s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1133 | max_forward: 0.9408 | max_loss: 0.1476 | max_backward: 1.8574 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 1.0209 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6146.8 | uss_MB: 5682.0 | child_rss_MB: 4174.9 | allocated_MB: 447.7 | reserved_MB: 31974.0 | inactive_split_MB: 968.4 +rank7 [2026-10-29 02:40:30,746] INFO : Step 17900/50000 rank7 | Train Loss: 1.0409 | Train Loss Unweighted: 0.5191 | Train Loss Components: Auxiliary: 0.2075 | Cardinality: 0.1356 | Classification: 0.0828 | Classification_binary: 0.1533 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1388 | Regression_eta: 0.0015 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0792 | Regression_eta: 0.0369 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0529 | LR: 7.44e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5932s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7095 | max_loss: 0.1607 | max_backward: 1.7215 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0394 | Memory: rss_MB: 6145.0 | uss_MB: 5678.8 | child_rss_MB: 4150.5 | allocated_MB: 445.2 | reserved_MB: 32302.0 | inactive_split_MB: 1177.5 +rank7 [2026-10-29 02:45:16,090] INFO : Step 18000/50000 rank7 | Train Loss: 1.1292 | Train Loss Unweighted: 0.5474 | Train Loss Components: Auxiliary: 0.2299 | Cardinality: 0.1496 | Classification: 0.0889 | Classification_binary: 0.1916 | HitGrouping: 0.1667 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0667 | Regression_eta: 0.0543 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0873 | LR: 7.42e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6576s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7753 | max_loss: 0.1523 | max_backward: 1.8844 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.8361 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 6145.1 | uss_MB: 5680.7 | child_rss_MB: 4175.2 | allocated_MB: 446.8 | reserved_MB: 32302.0 | inactive_split_MB: 1097.6 +rank7 [2026-10-29 02:45:17,190] INFO : Running validation on rank7 +rank7 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:45:17,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:46:12,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:46:13,322] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 02:46:39,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:46:40,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:46:41,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:46:42,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:46:52,538] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 02:46:53,116] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:46:53,609] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 02:47:09,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:47:10,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:47:12,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:47:14,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:47:24,058] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 02:47:25,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:47:25,822] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 02:47:41,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:47:42,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:47:43,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:47:44,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_18000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:47:54,114] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 02:53:14,468] INFO : Step 18100/50000 rank7 | Train Loss: 1.2236 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1598 | Classification: 0.1057 | Classification_binary: 0.2304 | HitGrouping: 0.1680 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0021 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2919 | Regression_cos_phi: 0.1035 | Regression_energy: 0.0949 | Regression_eta: 0.0695 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0890 | LR: 7.39e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7390s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8375 | max_loss: 0.1619 | max_backward: 1.6888 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.9700 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7199.0 | uss_MB: 6728.8 | child_rss_MB: 4164.6 | allocated_MB: 449.5 | reserved_MB: 25980.0 | inactive_split_MB: 1342.8 +rank7 [2026-10-29 02:58:08,645] INFO : Step 18200/50000 rank7 | Train Loss: 1.3226 | Train Loss Unweighted: 0.6675 | Train Loss Components: Auxiliary: 0.2616 | Cardinality: 0.1981 | Classification: 0.1030 | Classification_binary: 0.2037 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1777 | Regression_eta: 0.0019 | Regression_pt: 0.1789 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2368 | Regression_cos_phi: 0.1429 | Regression_energy: 0.1324 | Regression_eta: 0.0539 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0816 | LR: 7.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9662s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8334 | max_loss: 0.1467 | max_backward: 1.9856 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.9267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6214.2 | uss_MB: 5745.0 | child_rss_MB: 4152.2 | allocated_MB: 448.2 | reserved_MB: 29470.0 | inactive_split_MB: 891.6 +rank7 [2026-10-29 03:02:55,066] INFO : Step 18300/50000 rank7 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5446 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1133 | Classification: 0.0809 | Classification_binary: 0.1679 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1451 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0753 | Regression_eta: 0.0487 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0672 | LR: 7.34e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.5931s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7669 | max_loss: 0.6720 | max_backward: 2.7148 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 0.7387 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1510 | Memory: rss_MB: 6222.1 | uss_MB: 5756.1 | child_rss_MB: 4128.8 | allocated_MB: 450.0 | reserved_MB: 29474.0 | inactive_split_MB: 1394.0 +rank7 [2026-10-29 03:07:45,977] INFO : Step 18400/50000 rank7 | Train Loss: 1.2309 | Train Loss Unweighted: 0.5871 | Train Loss Components: Auxiliary: 0.2385 | Cardinality: 0.1783 | Classification: 0.0878 | Classification_binary: 0.2159 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1381 | Regression_eta: 0.0020 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2310 | Regression_cos_phi: 0.0978 | Regression_energy: 0.0712 | Regression_eta: 0.0653 | Regression_pt: 0.0736 | Regression_sin_phi: 0.1095 | LR: 7.31e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7129s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0450 | max_forward: 0.8287 | max_loss: 0.1526 | max_backward: 1.9572 | max_optimizer_step: 0.2153 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6218.0 | uss_MB: 5757.0 | child_rss_MB: 4115.3 | allocated_MB: 449.7 | reserved_MB: 29474.0 | inactive_split_MB: 952.9 +rank7 [2026-10-29 03:12:43,564] INFO : Step 18500/50000 rank7 | Train Loss: 1.1353 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1376 | Classification: 0.0933 | Classification_binary: 0.1928 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1375 | Regression_eta: 0.0018 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0639 | Regression_eta: 0.0411 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0442 | LR: 7.29e-04 | DataLoad Time: 0.0141s | Model Forward Time: 5.3593s | Phase Times: max_pre_step_barrier: 0.0403 | max_batch_to_device: 0.0065 | max_forward: 1.7657 | max_loss: 0.2172 | max_backward: 2.5100 | max_optimizer_step: 0.2152 | max_tensorboard_logging: 0.6220 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 6213.5 | uss_MB: 5757.9 | child_rss_MB: 4101.1 | allocated_MB: 449.3 | reserved_MB: 29476.0 | inactive_split_MB: 1334.3 +rank7 [2026-10-29 03:17:39,474] INFO : Step 18600/50000 rank7 | Train Loss: 1.1117 | Train Loss Unweighted: 0.5534 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1288 | Classification: 0.0848 | Classification_binary: 0.1638 | HitGrouping: 0.1712 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1498 | Regression_eta: 0.0021 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0596 | Regression_eta: 0.0857 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0590 | LR: 7.26e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.4731s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0164 | max_forward: 1.7588 | max_loss: 0.2357 | max_backward: 2.7448 | max_optimizer_step: 0.2193 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0807 | Memory: rss_MB: 6211.8 | uss_MB: 5757.0 | child_rss_MB: 4095.3 | allocated_MB: 447.4 | reserved_MB: 29476.0 | inactive_split_MB: 1348.2 +rank7 [2026-10-29 03:22:31,158] INFO : Step 18700/50000 rank7 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5820 | Train Loss Components: Auxiliary: 0.2213 | Cardinality: 0.1134 | Classification: 0.0931 | Classification_binary: 0.1880 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1481 | Regression_eta: 0.0018 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0608 | Regression_eta: 0.0386 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0612 | LR: 7.24e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4795s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.8092 | max_loss: 0.1474 | max_backward: 1.9113 | max_optimizer_step: 0.2170 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6208.9 | uss_MB: 5755.7 | child_rss_MB: 4068.3 | allocated_MB: 451.4 | reserved_MB: 31564.0 | inactive_split_MB: 952.4 +rank7 [2026-10-29 03:27:37,244] INFO : Step 18800/50000 rank7 | Train Loss: 1.2045 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1938 | Classification: 0.0840 | Classification_binary: 0.1934 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1412 | Regression_eta: 0.0018 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0705 | Regression_eta: 0.0379 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0459 | LR: 7.21e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.9533s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8320 | max_loss: 0.2425 | max_backward: 4.1386 | max_optimizer_step: 0.2694 | max_tensorboard_logging: 0.5740 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6207.7 | uss_MB: 5758.7 | child_rss_MB: 4075.2 | allocated_MB: 450.2 | reserved_MB: 31564.0 | inactive_split_MB: 1344.6 +rank7 [2026-10-29 03:32:26,944] INFO : Step 18900/50000 rank7 | Train Loss: 1.0005 | Train Loss Unweighted: 0.5202 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1059 | Classification: 0.0778 | Classification_binary: 0.1611 | HitGrouping: 0.1612 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1393 | Regression_eta: 0.0016 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0765 | Regression_eta: 0.0419 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0598 | LR: 7.18e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5380s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.7525 | max_loss: 0.1419 | max_backward: 1.7588 | max_optimizer_step: 0.3169 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6199.9 | uss_MB: 5753.2 | child_rss_MB: 4076.8 | allocated_MB: 448.0 | reserved_MB: 31564.0 | inactive_split_MB: 1008.7 +rank7 [2026-10-29 03:37:06,569] INFO : Step 19000/50000 rank7 | Train Loss: 1.1912 | Train Loss Unweighted: 0.5756 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1533 | Classification: 0.0950 | Classification_binary: 0.1836 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1461 | Regression_eta: 0.0020 | Regression_pt: 0.1464 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1820 | Regression_cos_phi: 0.1259 | Regression_energy: 0.0759 | Regression_eta: 0.0532 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0855 | LR: 7.16e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5313s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0061 | max_forward: 0.7769 | max_loss: 0.2439 | max_backward: 1.8831 | max_optimizer_step: 0.3053 | max_tensorboard_logging: 0.5996 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6201.1 | uss_MB: 5754.4 | child_rss_MB: 4071.5 | allocated_MB: 447.2 | reserved_MB: 31564.0 | inactive_split_MB: 1326.8 +rank7 [2026-10-29 03:37:07,713] INFO : Running validation on rank7 +rank7 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 03:37:07,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 03:38:00,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:38:00,821] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 03:38:26,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 03:38:28,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 03:38:29,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 03:38:30,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 03:38:40,383] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 03:38:40,771] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:38:41,426] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 03:38:57,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 03:38:58,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 03:38:59,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 03:39:00,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 03:39:10,337] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 03:39:10,608] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:39:11,226] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 03:39:26,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 03:39:27,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 03:39:28,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 03:39:29,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_19000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 03:39:39,285] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 03:44:42,722] INFO : Step 19100/50000 rank7 | Train Loss: 1.3611 | Train Loss Unweighted: 0.6138 | Train Loss Components: Auxiliary: 0.2635 | Cardinality: 0.2640 | Classification: 0.0924 | Classification_binary: 0.2317 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1410 | Regression_eta: 0.0021 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2593 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0842 | Regression_eta: 0.0741 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0900 | LR: 7.13e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9848s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8088 | max_loss: 0.1488 | max_backward: 2.1797 | max_optimizer_step: 0.2371 | max_tensorboard_logging: 0.6910 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1353 | Memory: rss_MB: 7402.3 | uss_MB: 6946.7 | child_rss_MB: 4070.8 | allocated_MB: 447.8 | reserved_MB: 26348.0 | inactive_split_MB: 1426.8 +rank7 [2026-10-29 03:49:23,547] INFO : Step 19200/50000 rank7 | Train Loss: 1.0281 | Train Loss Unweighted: 0.5045 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1380 | Classification: 0.0783 | Classification_binary: 0.1716 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0613 | Regression_eta: 0.0405 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0654 | LR: 7.10e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5619s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0059 | max_forward: 0.7822 | max_loss: 0.5179 | max_backward: 1.7218 | max_optimizer_step: 0.2343 | max_tensorboard_logging: 0.6971 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6283.3 | uss_MB: 5827.9 | child_rss_MB: 4069.2 | allocated_MB: 446.7 | reserved_MB: 30762.0 | inactive_split_MB: 1348.5 +rank7 [2026-10-29 03:54:09,159] INFO : Step 19300/50000 rank7 | Train Loss: 1.0218 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.1985 | Cardinality: 0.1217 | Classification: 0.0854 | Classification_binary: 0.1516 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0017 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0645 | Regression_eta: 0.0456 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0603 | LR: 7.08e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3959s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0901 | max_forward: 0.8291 | max_loss: 0.1410 | max_backward: 1.8901 | max_optimizer_step: 0.2184 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 6284.0 | uss_MB: 5839.5 | child_rss_MB: 4076.7 | allocated_MB: 445.8 | reserved_MB: 30762.0 | inactive_split_MB: 1181.3 +rank7 [2026-10-29 03:58:51,216] INFO : Step 19400/50000 rank7 | Train Loss: 1.2058 | Train Loss Unweighted: 0.6050 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1537 | Classification: 0.0974 | Classification_binary: 0.2074 | HitGrouping: 0.1694 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1473 | Regression_eta: 0.0019 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2158 | Regression_cos_phi: 0.1048 | Regression_energy: 0.0739 | Regression_eta: 0.0673 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0991 | LR: 7.05e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6150s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8033 | max_loss: 0.2086 | max_backward: 2.0346 | max_optimizer_step: 0.2196 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6285.1 | uss_MB: 5842.6 | child_rss_MB: 4066.7 | allocated_MB: 450.5 | reserved_MB: 30762.0 | inactive_split_MB: 1177.0 +rank7 [2026-10-29 04:03:30,642] INFO : Step 19500/50000 rank7 | Train Loss: 1.1473 | Train Loss Unweighted: 0.5713 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1373 | Classification: 0.0821 | Classification_binary: 0.2101 | HitGrouping: 0.1732 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0019 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0813 | Regression_eta: 0.0456 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0738 | LR: 7.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7159s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.7878 | max_loss: 0.1592 | max_backward: 1.7499 | max_optimizer_step: 0.2168 | max_tensorboard_logging: 0.7887 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6276.4 | uss_MB: 5837.3 | child_rss_MB: 4076.4 | allocated_MB: 448.1 | reserved_MB: 30762.0 | inactive_split_MB: 946.4 +rank7 [2026-10-29 04:08:12,465] INFO : Step 19600/50000 rank7 | Train Loss: 1.0237 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1141 | Classification: 0.0820 | Classification_binary: 0.1628 | HitGrouping: 0.1682 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1372 | Regression_eta: 0.0016 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0727 | Regression_eta: 0.0495 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0604 | LR: 7.00e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.4170s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.8452 | max_loss: 0.1424 | max_backward: 1.8495 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6275.8 | uss_MB: 5841.0 | child_rss_MB: 4055.7 | allocated_MB: 444.6 | reserved_MB: 30764.0 | inactive_split_MB: 1395.5 +rank7 [2026-10-29 04:12:52,347] INFO : Step 19700/50000 rank7 | Train Loss: 1.1272 | Train Loss Unweighted: 0.5572 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1372 | Classification: 0.0960 | Classification_binary: 0.1832 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1363 | Regression_eta: 0.0019 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1875 | Regression_cos_phi: 0.1255 | Regression_energy: 0.0688 | Regression_eta: 0.0827 | Regression_pt: 0.0709 | Regression_sin_phi: 0.1058 | LR: 6.97e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.3585s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0202 | max_forward: 0.8068 | max_loss: 0.1484 | max_backward: 1.8536 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.4841 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 6270.8 | uss_MB: 5840.8 | child_rss_MB: 4063.7 | allocated_MB: 452.5 | reserved_MB: 30764.0 | inactive_split_MB: 1379.9 +rank7 [2026-10-29 04:17:28,314] INFO : Step 19800/50000 rank7 | Train Loss: 1.1397 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1586 | Classification: 0.0822 | Classification_binary: 0.2154 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1276 | Regression_eta: 0.0016 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0900 | Regression_eta: 0.0373 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0821 | LR: 6.94e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.0466s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.1157 | max_forward: 0.8998 | max_loss: 0.2529 | max_backward: 2.2173 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.6920 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6270.7 | uss_MB: 5845.8 | child_rss_MB: 4080.0 | allocated_MB: 451.7 | reserved_MB: 32286.0 | inactive_split_MB: 1195.2 +rank7 [2026-10-29 04:22:08,870] INFO : Step 19900/50000 rank7 | Train Loss: 1.2749 | Train Loss Unweighted: 0.6377 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1657 | Classification: 0.1023 | Classification_binary: 0.1956 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1660 | Regression_eta: 0.0022 | Regression_pt: 0.1691 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0814 | Regression_eta: 0.0587 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0712 | LR: 6.92e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.3168s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.8477 | max_loss: 0.1438 | max_backward: 2.2730 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 1.0505 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 6257.5 | uss_MB: 5837.3 | child_rss_MB: 4080.7 | allocated_MB: 447.7 | reserved_MB: 32286.0 | inactive_split_MB: 1412.9 +rank7 [2026-10-29 04:26:51,973] INFO : Step 20000/50000 rank7 | Train Loss: 1.0534 | Train Loss Unweighted: 0.5290 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1287 | Classification: 0.0845 | Classification_binary: 0.1591 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1397 | Regression_eta: 0.0016 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0556 | Regression_eta: 0.0467 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0736 | LR: 6.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6553s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7878 | max_loss: 0.2534 | max_backward: 1.8478 | max_optimizer_step: 0.3086 | max_tensorboard_logging: 0.7553 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6260.9 | uss_MB: 5840.0 | child_rss_MB: 4050.1 | allocated_MB: 451.0 | reserved_MB: 32286.0 | inactive_split_MB: 1086.4 +rank7 [2026-10-29 04:26:52,970] INFO : Running validation on rank7 +rank7 [2026-10-29 04:26:52,970] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:26:52,971] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:27:46,035] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:27:46,652] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 04:28:14,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:28:15,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:28:16,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:28:17,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:28:27,263] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:28:27,955] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:28:28,500] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 04:28:44,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:28:45,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:28:46,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:28:47,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:28:57,793] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 04:28:58,729] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:28:59,248] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 04:29:14,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:29:15,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:29:16,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:29:17,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_20000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:29:27,338] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 04:34:27,056] INFO : Step 20100/50000 rank7 | Train Loss: 1.2003 | Train Loss Unweighted: 0.5926 | Train Loss Components: Auxiliary: 0.2453 | Cardinality: 0.1786 | Classification: 0.0941 | Classification_binary: 0.2103 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1419 | Regression_eta: 0.0017 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0949 | Regression_eta: 0.0614 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0683 | LR: 6.86e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.3263s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1130 | max_forward: 0.9438 | max_loss: 0.2499 | max_backward: 2.3678 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.7202 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.1025 | Memory: rss_MB: 7375.1 | uss_MB: 6951.0 | child_rss_MB: 4085.1 | allocated_MB: 451.2 | reserved_MB: 25134.0 | inactive_split_MB: 1371.4 +rank7 [2026-10-29 04:39:08,146] INFO : Step 20200/50000 rank7 | Train Loss: 1.0522 | Train Loss Unweighted: 0.5278 | Train Loss Components: Auxiliary: 0.2134 | Cardinality: 0.1298 | Classification: 0.0762 | Classification_binary: 0.1808 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1336 | Regression_eta: 0.0015 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0756 | Regression_eta: 0.0522 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0666 | LR: 6.84e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6155s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8072 | max_loss: 0.1481 | max_backward: 1.7933 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.7556 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0636 | Memory: rss_MB: 7367.2 | uss_MB: 6947.3 | child_rss_MB: 4028.7 | allocated_MB: 448.1 | reserved_MB: 26186.0 | inactive_split_MB: 1331.9 +rank7 [2026-10-29 04:43:47,103] INFO : Step 20300/50000 rank7 | Train Loss: 0.9658 | Train Loss Unweighted: 0.4644 | Train Loss Components: Auxiliary: 0.1856 | Cardinality: 0.1103 | Classification: 0.0804 | Classification_binary: 0.1497 | HitGrouping: 0.1647 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1154 | Regression_eta: 0.0013 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0299 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0894 | Regression_energy: 0.0732 | Regression_eta: 0.0347 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0880 | LR: 6.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4957s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.7923 | max_loss: 0.1397 | max_backward: 1.6644 | max_optimizer_step: 0.2986 | max_tensorboard_logging: 0.7635 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0492 | Memory: rss_MB: 6341.9 | uss_MB: 5926.8 | child_rss_MB: 4039.7 | allocated_MB: 446.7 | reserved_MB: 26186.0 | inactive_split_MB: 1009.7 +rank7 [2026-10-29 04:48:26,731] INFO : Step 20400/50000 rank7 | Train Loss: 1.1021 | Train Loss Unweighted: 0.5603 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1341 | Classification: 0.0914 | Classification_binary: 0.1899 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.1798 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0659 | Regression_eta: 0.0473 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0699 | LR: 6.78e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4425s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0078 | max_forward: 0.8020 | max_loss: 0.1500 | max_backward: 1.6276 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7742 | max_tensorboard_flush: 0.0146 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6341.7 | uss_MB: 5926.6 | child_rss_MB: 4040.0 | allocated_MB: 450.1 | reserved_MB: 28222.0 | inactive_split_MB: 1363.9 +rank7 [2026-10-29 04:53:08,466] INFO : Step 20500/50000 rank7 | Train Loss: 1.1614 | Train Loss Unweighted: 0.5764 | Train Loss Components: Auxiliary: 0.2298 | Cardinality: 0.1384 | Classification: 0.0955 | Classification_binary: 0.2119 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1313 | Regression_eta: 0.0021 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0797 | Regression_eta: 0.0545 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0811 | LR: 6.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0621s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.9215 | max_loss: 0.1527 | max_backward: 2.1638 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.8562 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0480 | Memory: rss_MB: 6338.4 | uss_MB: 5929.0 | child_rss_MB: 4027.5 | allocated_MB: 448.1 | reserved_MB: 28224.0 | inactive_split_MB: 1364.4 +rank7 [2026-10-29 04:57:53,290] INFO : Step 20600/50000 rank7 | Train Loss: 1.0112 | Train Loss Unweighted: 0.5058 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1291 | Classification: 0.0712 | Classification_binary: 0.1592 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1355 | Regression_eta: 0.0016 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0724 | Regression_eta: 0.0470 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0787 | LR: 6.73e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7536s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.8345 | max_loss: 0.1445 | max_backward: 2.0143 | max_optimizer_step: 0.2178 | max_tensorboard_logging: 0.5368 | max_tensorboard_flush: 0.0063 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6335.2 | uss_MB: 5762.5 | child_rss_MB: 4012.2 | allocated_MB: 447.1 | reserved_MB: 29088.0 | inactive_split_MB: 1370.9 +rank7 [2026-10-29 05:02:47,889] INFO : Step 20700/50000 rank7 | Train Loss: 1.0948 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1460 | Classification: 0.0979 | Classification_binary: 0.1794 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1251 | Regression_eta: 0.0018 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0667 | Regression_eta: 0.0378 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0556 | LR: 6.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6381s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7758 | max_loss: 0.1773 | max_backward: 1.7778 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8037 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 6333.8 | uss_MB: 5926.6 | child_rss_MB: 4020.5 | allocated_MB: 448.0 | reserved_MB: 29088.0 | inactive_split_MB: 1374.9 +rank7 [2026-10-29 05:07:34,619] INFO : Step 20800/50000 rank7 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5517 | Train Loss Components: Auxiliary: 0.2356 | Cardinality: 0.1461 | Classification: 0.0813 | Classification_binary: 0.2082 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0740 | Regression_eta: 0.0466 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0505 | LR: 6.67e-04 | DataLoad Time: 0.0192s | Model Forward Time: 6.8149s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.1047 | max_forward: 1.3297 | max_loss: 0.2564 | max_backward: 3.5637 | max_optimizer_step: 0.2186 | max_tensorboard_logging: 0.6607 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1311 | Memory: rss_MB: 6333.6 | uss_MB: 5928.9 | child_rss_MB: 4021.2 | allocated_MB: 447.7 | reserved_MB: 29088.0 | inactive_split_MB: 1366.9 +rank7 [2026-10-29 05:12:18,487] INFO : Step 20900/50000 rank7 | Train Loss: 0.9640 | Train Loss Unweighted: 0.4835 | Train Loss Components: Auxiliary: 0.1915 | Cardinality: 0.1180 | Classification: 0.0745 | Classification_binary: 0.1573 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1243 | Regression_eta: 0.0014 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0641 | Regression_eta: 0.0292 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0455 | LR: 6.65e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6285s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0819 | max_forward: 0.8427 | max_loss: 0.1415 | max_backward: 1.9292 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 6333.3 | uss_MB: 5928.3 | child_rss_MB: 4019.0 | allocated_MB: 447.3 | reserved_MB: 29088.0 | inactive_split_MB: 1171.1 +rank7 [2026-10-29 05:17:03,997] INFO : Step 21000/50000 rank7 | Train Loss: 1.0251 | Train Loss Unweighted: 0.4983 | Train Loss Components: Auxiliary: 0.2063 | Cardinality: 0.1326 | Classification: 0.0792 | Classification_binary: 0.1531 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1308 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0707 | Regression_eta: 0.0449 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0593 | LR: 6.62e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5925s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7480 | max_loss: 0.2499 | max_backward: 1.8837 | max_optimizer_step: 0.3173 | max_tensorboard_logging: 0.6559 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6332.4 | uss_MB: 5927.5 | child_rss_MB: 4014.2 | allocated_MB: 448.7 | reserved_MB: 29088.0 | inactive_split_MB: 1177.0 +rank7 [2026-10-29 05:17:04,884] INFO : Running validation on rank7 +rank7 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 05:17:04,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 05:17:58,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:17:58,800] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 05:18:22,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 05:18:23,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 05:18:24,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 05:18:25,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 05:18:35,929] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 05:18:36,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:18:37,118] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 05:18:53,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 05:18:54,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 05:18:55,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 05:18:56,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 05:19:06,266] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 05:19:06,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:19:07,300] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 05:19:22,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 05:19:23,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 05:19:24,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 05:19:25,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_21000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 05:19:35,090] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 05:24:38,578] INFO : Step 21100/50000 rank7 | Train Loss: 1.1711 | Train Loss Unweighted: 0.5509 | Train Loss Components: Auxiliary: 0.2351 | Cardinality: 0.1728 | Classification: 0.0904 | Classification_binary: 0.2017 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1264 | Regression_eta: 0.0019 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2246 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0662 | Regression_eta: 0.0519 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0751 | LR: 6.59e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.3804s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1509 | max_backward: 2.3487 | max_optimizer_step: 0.2398 | max_tensorboard_logging: 0.8777 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 7573.7 | uss_MB: 6895.0 | child_rss_MB: 4017.0 | allocated_MB: 449.6 | reserved_MB: 23886.0 | inactive_split_MB: 970.9 +rank7 [2026-10-29 05:25:02,180] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank7 [2026-10-29 05:25:02,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 05:29:24,523] INFO : Step 21200/50000 rank7 | Train Loss: 0.9890 | Train Loss Unweighted: 0.5032 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.1158 | Classification: 0.0766 | Classification_binary: 0.1454 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1377 | Regression_eta: 0.0015 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0811 | Regression_eta: 0.0426 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0943 | LR: 6.56e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6424s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8130 | max_loss: 0.1392 | max_backward: 2.0783 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1439 | Memory: rss_MB: 7574.1 | uss_MB: 7157.5 | child_rss_MB: 4009.6 | allocated_MB: 450.6 | reserved_MB: 25902.0 | inactive_split_MB: 1031.5 +rank7 [2026-10-29 05:34:39,116] INFO : Step 21300/50000 rank7 | Train Loss: 1.0430 | Train Loss Unweighted: 0.5133 | Train Loss Components: Auxiliary: 0.2131 | Cardinality: 0.1363 | Classification: 0.0760 | Classification_binary: 0.1735 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1294 | Regression_eta: 0.0016 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0699 | Regression_eta: 0.0373 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0652 | LR: 6.54e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6730s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7638 | max_loss: 0.2151 | max_backward: 1.9054 | max_optimizer_step: 0.3055 | max_tensorboard_logging: 0.7415 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 6440.0 | uss_MB: 6027.2 | child_rss_MB: 4019.8 | allocated_MB: 448.6 | reserved_MB: 25966.0 | inactive_split_MB: 1356.4 +rank7 [2026-10-29 05:39:29,689] INFO : Step 21400/50000 rank7 | Train Loss: 1.1932 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1665 | Classification: 0.0905 | Classification_binary: 0.2123 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0654 | Regression_eta: 0.0571 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0631 | LR: 6.51e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.3959s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.0713 | max_loss: 0.5046 | max_backward: 2.7133 | max_optimizer_step: 0.2318 | max_tensorboard_logging: 0.5348 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6442.2 | uss_MB: 6029.7 | child_rss_MB: 4021.3 | allocated_MB: 450.1 | reserved_MB: 25966.0 | inactive_split_MB: 1418.5 +rank7 [2026-10-29 05:44:17,735] INFO : Step 21500/50000 rank7 | Train Loss: 0.9943 | Train Loss Unweighted: 0.4774 | Train Loss Components: Auxiliary: 0.1990 | Cardinality: 0.1328 | Classification: 0.0688 | Classification_binary: 0.1534 | HitGrouping: 0.1658 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1250 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0627 | Regression_eta: 0.0350 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0872 | LR: 6.48e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3075s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7931 | max_loss: 0.1447 | max_backward: 1.5906 | max_optimizer_step: 0.3210 | max_tensorboard_logging: 0.6035 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6439.8 | uss_MB: 6027.9 | child_rss_MB: 4005.9 | allocated_MB: 444.2 | reserved_MB: 25966.0 | inactive_split_MB: 998.2 +rank7 [2026-10-29 05:49:11,032] INFO : Step 21600/50000 rank7 | Train Loss: 1.0570 | Train Loss Unweighted: 0.5255 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1241 | Classification: 0.0816 | Classification_binary: 0.1687 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1353 | Regression_eta: 0.0015 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0617 | Regression_eta: 0.0486 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0633 | LR: 6.45e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5087s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7771 | max_loss: 0.1495 | max_backward: 1.8938 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 0.6004 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 6439.3 | uss_MB: 6031.0 | child_rss_MB: 3996.2 | allocated_MB: 449.1 | reserved_MB: 25966.0 | inactive_split_MB: 1354.8 +rank7 [2026-10-29 05:53:59,565] INFO : Step 21700/50000 rank7 | Train Loss: 1.1491 | Train Loss Unweighted: 0.5692 | Train Loss Components: Auxiliary: 0.2320 | Cardinality: 0.1414 | Classification: 0.0964 | Classification_binary: 0.2079 | HitGrouping: 0.1613 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1295 | Regression_eta: 0.0019 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2061 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0692 | Regression_eta: 0.0538 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0718 | LR: 6.43e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.5552s | Phase Times: max_pre_step_barrier: 0.1087 | max_batch_to_device: 0.0069 | max_forward: 0.8265 | max_loss: 0.2302 | max_backward: 2.2344 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 1.1494 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6433.9 | uss_MB: 4869.4 | child_rss_MB: 3996.2 | allocated_MB: 449.0 | reserved_MB: 29346.0 | inactive_split_MB: 1158.8 +rank7 [2026-10-29 05:58:52,460] INFO : Step 21800/50000 rank7 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4969 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1122 | Classification: 0.0717 | Classification_binary: 0.1743 | HitGrouping: 0.1609 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0750 | Regression_eta: 0.0388 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0556 | LR: 6.40e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1494s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0060 | max_forward: 1.1202 | max_loss: 0.2147 | max_backward: 2.1549 | max_optimizer_step: 0.3223 | max_tensorboard_logging: 0.7959 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6428.8 | uss_MB: 6028.1 | child_rss_MB: 3976.6 | allocated_MB: 445.7 | reserved_MB: 29346.0 | inactive_split_MB: 1029.4 +rank7 [2026-10-29 06:03:49,407] INFO : Step 21900/50000 rank7 | Train Loss: 1.0489 | Train Loss Unweighted: 0.5440 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1253 | Classification: 0.0833 | Classification_binary: 0.1514 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1521 | Regression_eta: 0.0016 | Regression_pt: 0.1538 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0751 | Regression_eta: 0.0394 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0719 | LR: 6.37e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8515s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.8475 | max_loss: 0.1424 | max_backward: 2.0460 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.7894 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0780 | Memory: rss_MB: 6406.9 | uss_MB: 6029.1 | child_rss_MB: 4002.3 | allocated_MB: 445.6 | reserved_MB: 29346.0 | inactive_split_MB: 1036.5 +rank7 [2026-10-29 06:08:50,709] INFO : Step 22000/50000 rank7 | Train Loss: 1.2256 | Train Loss Unweighted: 0.5955 | Train Loss Components: Auxiliary: 0.2502 | Cardinality: 0.1814 | Classification: 0.1004 | Classification_binary: 0.2101 | HitGrouping: 0.1678 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1388 | Regression_eta: 0.0018 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0715 | Regression_eta: 0.0428 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0666 | LR: 6.34e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4731s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.7995 | max_loss: 0.3048 | max_backward: 2.4135 | max_optimizer_step: 0.2299 | max_tensorboard_logging: 0.9636 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0487 | Memory: rss_MB: 6406.0 | uss_MB: 5842.6 | child_rss_MB: 4004.6 | allocated_MB: 446.4 | reserved_MB: 29346.0 | inactive_split_MB: 1354.8 +rank7 [2026-10-29 06:08:51,806] INFO : Running validation on rank7 +rank7 [2026-10-29 06:08:51,806] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 06:08:51,806] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 06:09:45,572] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:09:46,125] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 06:10:11,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 06:10:12,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 06:10:14,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 06:10:15,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 06:10:25,274] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 06:10:26,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:10:27,417] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 06:10:43,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 06:10:44,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 06:10:45,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 06:10:46,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 06:10:56,529] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 06:10:56,848] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:10:57,441] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 06:11:13,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 06:11:14,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 06:11:14,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 06:11:15,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_22000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 06:11:25,646] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 06:16:42,858] INFO : Step 22100/50000 rank7 | Train Loss: 1.1574 | Train Loss Unweighted: 0.5647 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1483 | Classification: 0.0852 | Classification_binary: 0.2153 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1296 | Regression_eta: 0.0018 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0622 | Regression_eta: 0.0452 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0895 | LR: 6.32e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1894s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.8412 | max_loss: 0.5331 | max_backward: 1.9944 | max_optimizer_step: 0.3227 | max_tensorboard_logging: 1.0038 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 7529.1 | uss_MB: 7150.1 | child_rss_MB: 3978.3 | allocated_MB: 449.5 | reserved_MB: 25606.0 | inactive_split_MB: 1092.5 +rank7 [2026-10-29 06:21:31,722] INFO : Step 22200/50000 rank7 | Train Loss: 0.9886 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.1997 | Cardinality: 0.1142 | Classification: 0.0851 | Classification_binary: 0.1469 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1291 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0658 | Regression_eta: 0.0559 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0947 | LR: 6.29e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.2918s | Phase Times: max_pre_step_barrier: 0.0055 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.1602 | max_backward: 2.2825 | max_optimizer_step: 0.2420 | max_tensorboard_logging: 0.9470 | max_tensorboard_flush: 0.0165 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 7529.9 | uss_MB: 5524.9 | child_rss_MB: 3953.0 | allocated_MB: 446.3 | reserved_MB: 27282.0 | inactive_split_MB: 966.9 +rank7 [2026-10-29 06:26:33,414] INFO : Step 22300/50000 rank7 | Train Loss: 0.9955 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1158 | Classification: 0.0756 | Classification_binary: 0.1689 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0800 | Regression_eta: 0.0397 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0660 | LR: 6.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7534s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7779 | max_loss: 0.2096 | max_backward: 1.8547 | max_optimizer_step: 0.3294 | max_tensorboard_logging: 0.8021 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6493.2 | uss_MB: 6116.2 | child_rss_MB: 3947.3 | allocated_MB: 447.7 | reserved_MB: 27528.0 | inactive_split_MB: 1196.9 +rank7 [2026-10-29 06:31:29,808] INFO : Step 22400/50000 rank7 | Train Loss: 1.0741 | Train Loss Unweighted: 0.5341 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1284 | Classification: 0.0825 | Classification_binary: 0.2044 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1212 | Regression_eta: 0.0016 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0490 | Regression_eta: 0.0462 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0608 | LR: 6.23e-04 | DataLoad Time: 0.0113s | Model Forward Time: 6.3943s | Phase Times: max_pre_step_barrier: 0.0148 | max_batch_to_device: 0.0061 | max_forward: 1.2953 | max_loss: 0.2367 | max_backward: 3.1918 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.9806 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6488.3 | uss_MB: 5713.4 | child_rss_MB: 3943.1 | allocated_MB: 448.0 | reserved_MB: 27528.0 | inactive_split_MB: 1385.4 +rank7 [2026-10-29 06:36:31,514] INFO : Step 22500/50000 rank7 | Train Loss: 1.0224 | Train Loss Unweighted: 0.5017 | Train Loss Components: Auxiliary: 0.2029 | Cardinality: 0.1292 | Classification: 0.0752 | Classification_binary: 0.1565 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1329 | Regression_eta: 0.0015 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1361 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0764 | Regression_eta: 0.0366 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0504 | LR: 6.20e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2543s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0256 | max_forward: 1.0490 | max_loss: 0.1960 | max_backward: 2.3241 | max_optimizer_step: 0.2218 | max_tensorboard_logging: 0.8675 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6483.3 | uss_MB: 6114.9 | child_rss_MB: 3947.6 | allocated_MB: 446.0 | reserved_MB: 27530.0 | inactive_split_MB: 1333.8 +rank7 [2026-10-29 06:41:27,999] INFO : Step 22600/50000 rank7 | Train Loss: 1.0006 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.1172 | Classification: 0.0776 | Classification_binary: 0.1539 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1237 | Regression_eta: 0.0017 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0640 | Regression_eta: 0.0402 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0591 | LR: 6.18e-04 | DataLoad Time: 0.0253s | Model Forward Time: 4.3119s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.7639 | max_loss: 0.1436 | max_backward: 1.7983 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5297 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6476.1 | uss_MB: 6115.2 | child_rss_MB: 3945.2 | allocated_MB: 446.7 | reserved_MB: 27558.0 | inactive_split_MB: 937.3 +rank7 [2026-10-29 06:46:29,195] INFO : Step 22700/50000 rank7 | Train Loss: 1.1678 | Train Loss Unweighted: 0.5409 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1880 | Classification: 0.0891 | Classification_binary: 0.2014 | HitGrouping: 0.1625 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2194 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0578 | Regression_eta: 0.0447 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0751 | LR: 6.15e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9427s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0958 | max_forward: 1.3859 | max_loss: 0.3709 | max_backward: 2.0801 | max_optimizer_step: 0.2588 | max_tensorboard_logging: 0.6180 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6478.3 | uss_MB: 6114.3 | child_rss_MB: 3942.9 | allocated_MB: 449.6 | reserved_MB: 27558.0 | inactive_split_MB: 1187.5 +rank7 [2026-10-29 06:51:26,820] INFO : Step 22800/50000 rank7 | Train Loss: 1.0584 | Train Loss Unweighted: 0.5164 | Train Loss Components: Auxiliary: 0.2135 | Cardinality: 0.1296 | Classification: 0.0756 | Classification_binary: 0.1788 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1288 | Regression_eta: 0.0018 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0645 | Regression_eta: 0.0462 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0633 | LR: 6.12e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.4060s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0071 | max_forward: 0.9762 | max_loss: 0.5417 | max_backward: 2.3851 | max_optimizer_step: 0.2261 | max_tensorboard_logging: 1.0186 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6482.6 | uss_MB: 4953.7 | child_rss_MB: 3942.6 | allocated_MB: 450.1 | reserved_MB: 27562.0 | inactive_split_MB: 1063.4 +rank7 [2026-10-29 06:56:23,354] INFO : Step 22900/50000 rank7 | Train Loss: 0.9247 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.0985 | Classification: 0.0773 | Classification_binary: 0.1369 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0673 | Regression_eta: 0.0579 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0676 | LR: 6.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8108s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.7953 | max_loss: 0.1664 | max_backward: 1.8057 | max_optimizer_step: 0.3142 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0515 | Memory: rss_MB: 6474.9 | uss_MB: 6107.5 | child_rss_MB: 3942.0 | allocated_MB: 449.0 | reserved_MB: 29566.0 | inactive_split_MB: 1182.4 +rank7 [2026-10-29 07:01:33,062] INFO : Step 23000/50000 rank7 | Train Loss: 1.1236 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2234 | Cardinality: 0.1470 | Classification: 0.0818 | Classification_binary: 0.1979 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1282 | Regression_eta: 0.0019 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0659 | Regression_eta: 0.0653 | Regression_pt: 0.0676 | Regression_sin_phi: 0.1025 | LR: 6.06e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.4758s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0128 | max_forward: 0.8358 | max_loss: 0.1538 | max_backward: 2.0800 | max_optimizer_step: 0.3325 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0046 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 6480.2 | uss_MB: 5867.9 | child_rss_MB: 3944.5 | allocated_MB: 448.7 | reserved_MB: 29568.0 | inactive_split_MB: 927.2 +rank7 [2026-10-29 07:01:34,104] INFO : Running validation on rank7 +rank7 [2026-10-29 07:01:34,104] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:01:34,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:02:28,432] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:02:28,852] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 07:02:49,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:02:50,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:02:51,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:02:52,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:03:02,578] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:03:03,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:03:03,704] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 07:03:19,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:03:20,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:03:21,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:03:22,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:03:32,982] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:03:33,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:03:34,171] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 07:03:49,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:03:50,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:03:51,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:03:52,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_23000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:04:02,418] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 07:09:13,970] INFO : Step 23100/50000 rank7 | Train Loss: 1.1091 | Train Loss Unweighted: 0.5415 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1351 | Classification: 0.0839 | Classification_binary: 0.1998 | HitGrouping: 0.1641 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1259 | Regression_eta: 0.0018 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0312 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0677 | Regression_eta: 0.0349 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0560 | LR: 6.04e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1781s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.9011 | max_loss: 0.1541 | max_backward: 2.1118 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7511.1 | uss_MB: 7142.6 | child_rss_MB: 3950.6 | allocated_MB: 447.0 | reserved_MB: 26386.0 | inactive_split_MB: 1389.4 +rank7 [2026-10-29 07:14:12,717] INFO : Step 23200/50000 rank7 | Train Loss: 0.9623 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1158 | Classification: 0.0816 | Classification_binary: 0.1457 | HitGrouping: 0.1600 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1188 | Regression_eta: 0.0014 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0604 | Regression_eta: 0.0471 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0568 | LR: 6.01e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.6099s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0878 | max_forward: 0.8818 | max_loss: 0.1439 | max_backward: 1.8275 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0717 | Memory: rss_MB: 7510.7 | uss_MB: 7141.7 | child_rss_MB: 3950.6 | allocated_MB: 445.1 | reserved_MB: 26660.0 | inactive_split_MB: 1277.1 +rank7 [2026-10-29 07:19:05,797] INFO : Step 23300/50000 rank7 | Train Loss: 1.0259 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1150 | Classification: 0.0866 | Classification_binary: 0.1685 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0510 | Regression_eta: 0.0456 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0612 | LR: 5.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8581s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8107 | max_loss: 0.2480 | max_backward: 2.0694 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.7208 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 6508.5 | uss_MB: 6170.5 | child_rss_MB: 3889.6 | allocated_MB: 447.8 | reserved_MB: 28416.0 | inactive_split_MB: 1122.1 +rank7 [2026-10-29 07:24:01,132] INFO : Step 23400/50000 rank7 | Train Loss: 1.0554 | Train Loss Unweighted: 0.5231 | Train Loss Components: Auxiliary: 0.2137 | Cardinality: 0.1145 | Classification: 0.0843 | Classification_binary: 0.1879 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1229 | Regression_eta: 0.0017 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1905 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0662 | Regression_eta: 0.0453 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0677 | LR: 5.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0095s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8755 | max_loss: 0.1586 | max_backward: 2.0537 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.9298 | max_tensorboard_flush: 0.0045 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6503.8 | uss_MB: 6169.0 | child_rss_MB: 3888.5 | allocated_MB: 453.6 | reserved_MB: 30468.0 | inactive_split_MB: 1031.4 +rank7 [2026-10-29 07:29:14,710] INFO : Step 23500/50000 rank7 | Train Loss: 1.0199 | Train Loss Unweighted: 0.4883 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1355 | Classification: 0.0727 | Classification_binary: 0.1709 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0015 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0648 | Regression_eta: 0.0394 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0691 | LR: 5.92e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8321s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7958 | max_loss: 0.1474 | max_backward: 1.8900 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.7977 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0524 | Memory: rss_MB: 6504.5 | uss_MB: 6167.8 | child_rss_MB: 3892.2 | allocated_MB: 448.0 | reserved_MB: 30468.0 | inactive_split_MB: 1379.6 +rank7 [2026-10-29 07:34:22,318] INFO : Step 23600/50000 rank7 | Train Loss: 0.9927 | Train Loss Unweighted: 0.5038 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1159 | Classification: 0.0823 | Classification_binary: 0.1409 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1389 | Regression_eta: 0.0016 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0592 | Regression_eta: 0.0339 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0541 | LR: 5.90e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.3893s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0066 | max_forward: 1.3726 | max_loss: 0.3862 | max_backward: 2.5426 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.7444 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0514 | Memory: rss_MB: 6505.4 | uss_MB: 6168.6 | child_rss_MB: 3883.9 | allocated_MB: 447.7 | reserved_MB: 30468.0 | inactive_split_MB: 1182.0 +rank7 [2026-10-29 07:39:40,892] INFO : Step 23700/50000 rank7 | Train Loss: 1.0590 | Train Loss Unweighted: 0.5083 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1246 | Classification: 0.0809 | Classification_binary: 0.1840 | HitGrouping: 0.1598 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1190 | Regression_eta: 0.0018 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0693 | Regression_eta: 0.0563 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0716 | LR: 5.87e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.8154s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0068 | max_forward: 1.3226 | max_loss: 0.3023 | max_backward: 2.5425 | max_optimizer_step: 0.3121 | max_tensorboard_logging: 1.0647 | max_tensorboard_flush: 0.0197 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6501.5 | uss_MB: 6168.4 | child_rss_MB: 3886.7 | allocated_MB: 450.8 | reserved_MB: 31728.0 | inactive_split_MB: 1194.9 +rank7 [2026-10-29 07:44:46,683] INFO : Step 23800/50000 rank7 | Train Loss: 1.0874 | Train Loss Unweighted: 0.5345 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1265 | Classification: 0.0714 | Classification_binary: 0.1958 | HitGrouping: 0.1643 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1310 | Regression_eta: 0.0019 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0691 | Regression_eta: 0.0467 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0903 | LR: 5.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8769s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0445 | max_forward: 0.8564 | max_loss: 0.1471 | max_backward: 1.9401 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.8557 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6504.7 | uss_MB: 6170.9 | child_rss_MB: 3887.1 | allocated_MB: 450.5 | reserved_MB: 31728.0 | inactive_split_MB: 1115.9 +rank7 [2026-10-29 07:49:49,922] INFO : Step 23900/50000 rank7 | Train Loss: 0.9618 | Train Loss Unweighted: 0.4759 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1106 | Classification: 0.0766 | Classification_binary: 0.1436 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1261 | Regression_eta: 0.0014 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0623 | Regression_eta: 0.0388 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0585 | LR: 5.81e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8160s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.7901 | max_loss: 0.1437 | max_backward: 1.9199 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.8731 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0715 | Memory: rss_MB: 6494.0 | uss_MB: 6160.5 | child_rss_MB: 3895.1 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1201.1 +rank7 [2026-10-29 07:54:58,899] INFO : Step 24000/50000 rank7 | Train Loss: 1.0214 | Train Loss Unweighted: 0.4973 | Train Loss Components: Auxiliary: 0.2076 | Cardinality: 0.1221 | Classification: 0.0863 | Classification_binary: 0.1671 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1199 | Regression_eta: 0.0015 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0568 | Regression_eta: 0.0496 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0694 | LR: 5.78e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8555s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0359 | max_forward: 0.8595 | max_loss: 0.1545 | max_backward: 1.8284 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.9476 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6500.0 | uss_MB: 6167.8 | child_rss_MB: 3858.4 | allocated_MB: 448.1 | reserved_MB: 31728.0 | inactive_split_MB: 1361.9 +rank7 [2026-10-29 07:54:59,832] INFO : Running validation on rank7 +rank7 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:54:59,832] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:55:53,571] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:55:54,112] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 07:56:20,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:56:21,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:56:23,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:56:24,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:56:34,095] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:56:34,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:56:35,051] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 07:56:51,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:56:52,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:56:53,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:56:54,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:57:04,438] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 07:57:05,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:57:05,694] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 07:57:20,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:57:21,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:57:22,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:57:23,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_24000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:57:33,373] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 08:03:04,663] INFO : Step 24100/50000 rank7 | Train Loss: 1.1053 | Train Loss Unweighted: 0.5429 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1369 | Classification: 0.0857 | Classification_binary: 0.1997 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1266 | Regression_eta: 0.0017 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0761 | Regression_eta: 0.0362 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0612 | LR: 5.75e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5901s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8465 | max_loss: 0.1510 | max_backward: 2.0087 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 1.1897 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7733.0 | uss_MB: 6057.8 | child_rss_MB: 3871.6 | allocated_MB: 449.7 | reserved_MB: 24460.0 | inactive_split_MB: 1400.0 +rank7 [2026-10-29 08:07:59,793] INFO : Step 24200/50000 rank7 | Train Loss: 1.0082 | Train Loss Unweighted: 0.5034 | Train Loss Components: Auxiliary: 0.2055 | Cardinality: 0.1226 | Classification: 0.0813 | Classification_binary: 0.1571 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1306 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0752 | Regression_eta: 0.0506 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0616 | LR: 5.73e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0939s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0166 | max_forward: 0.7954 | max_loss: 0.1464 | max_backward: 1.7899 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.8196 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7726.9 | uss_MB: 6762.1 | child_rss_MB: 3882.9 | allocated_MB: 444.2 | reserved_MB: 26498.0 | inactive_split_MB: 1220.7 +rank7 [2026-10-29 08:13:18,505] INFO : Step 24300/50000 rank7 | Train Loss: 0.9673 | Train Loss Unweighted: 0.4938 | Train Loss Components: Auxiliary: 0.1960 | Cardinality: 0.0996 | Classification: 0.0803 | Classification_binary: 0.1488 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0679 | Regression_eta: 0.0369 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0778 | LR: 5.70e-04 | DataLoad Time: 0.0220s | Model Forward Time: 7.4879s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0743 | max_forward: 0.7911 | max_loss: 0.1390 | max_backward: 4.2004 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.9881 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0616 | Memory: rss_MB: 6561.7 | uss_MB: 6100.2 | child_rss_MB: 3885.3 | allocated_MB: 446.8 | reserved_MB: 26502.0 | inactive_split_MB: 1180.0 +rank7 [2026-10-29 08:18:15,798] INFO : Step 24400/50000 rank7 | Train Loss: 1.2095 | Train Loss Unweighted: 0.5558 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.2036 | Classification: 0.0907 | Classification_binary: 0.1975 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0021 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0719 | Regression_eta: 0.0529 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0717 | LR: 5.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1594s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0121 | max_forward: 0.9077 | max_loss: 0.1545 | max_backward: 2.1578 | max_optimizer_step: 0.2272 | max_tensorboard_logging: 0.9235 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 6562.6 | uss_MB: 6248.2 | child_rss_MB: 3881.5 | allocated_MB: 451.0 | reserved_MB: 26504.0 | inactive_split_MB: 1005.0 +rank7 [2026-10-29 08:23:16,668] INFO : Step 24500/50000 rank7 | Train Loss: 0.9848 | Train Loss Unweighted: 0.4865 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1186 | Classification: 0.0791 | Classification_binary: 0.1690 | HitGrouping: 0.1603 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1172 | Regression_eta: 0.0017 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0563 | Regression_eta: 0.0412 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0557 | LR: 5.64e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.5200s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.8328 | max_loss: 0.1536 | max_backward: 2.2930 | max_optimizer_step: 0.2628 | max_tensorboard_logging: 0.9058 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6554.5 | uss_MB: 6045.6 | child_rss_MB: 3882.4 | allocated_MB: 450.2 | reserved_MB: 26506.0 | inactive_split_MB: 1022.2 +rank7 [2026-10-29 08:28:18,514] INFO : Step 24600/50000 rank7 | Train Loss: 0.9612 | Train Loss Unweighted: 0.4788 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1053 | Classification: 0.0698 | Classification_binary: 0.1419 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0017 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0184 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0599 | Regression_eta: 0.0374 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0563 | LR: 5.61e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7039s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0058 | max_forward: 0.8192 | max_loss: 0.1791 | max_backward: 1.8073 | max_optimizer_step: 0.3091 | max_tensorboard_logging: 0.8063 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 6558.2 | uss_MB: 6245.9 | child_rss_MB: 3880.4 | allocated_MB: 445.6 | reserved_MB: 26506.0 | inactive_split_MB: 1452.5 +rank7 [2026-10-29 08:33:21,874] INFO : Step 24700/50000 rank7 | Train Loss: 1.0976 | Train Loss Unweighted: 0.5316 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1342 | Classification: 0.0931 | Classification_binary: 0.1856 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1235 | Regression_eta: 0.0021 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0619 | Regression_eta: 0.0463 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0661 | LR: 5.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9279s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8370 | max_loss: 0.1554 | max_backward: 1.9345 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.9388 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 6554.2 | uss_MB: 6245.2 | child_rss_MB: 3874.8 | allocated_MB: 453.9 | reserved_MB: 26508.0 | inactive_split_MB: 1444.9 +rank7 [2026-10-29 08:38:19,545] INFO : Step 24800/50000 rank7 | Train Loss: 1.0484 | Train Loss Unweighted: 0.5152 | Train Loss Components: Auxiliary: 0.2147 | Cardinality: 0.1288 | Classification: 0.0809 | Classification_binary: 0.1879 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1205 | Regression_eta: 0.0017 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0604 | Regression_eta: 0.0459 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0736 | LR: 5.56e-04 | DataLoad Time: 0.0227s | Model Forward Time: 5.2064s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.9054 | max_loss: 0.2650 | max_backward: 2.0866 | max_optimizer_step: 0.2323 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0620 | Memory: rss_MB: 6554.1 | uss_MB: 6245.6 | child_rss_MB: 3869.9 | allocated_MB: 449.5 | reserved_MB: 26508.0 | inactive_split_MB: 1443.3 +rank7 [2026-10-29 08:43:21,523] INFO : Step 24900/50000 rank7 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4587 | Train Loss Components: Auxiliary: 0.1835 | Cardinality: 0.0956 | Classification: 0.0744 | Classification_binary: 0.1418 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0670 | Regression_eta: 0.0342 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0716 | LR: 5.53e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4723s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0060 | max_forward: 0.7487 | max_loss: 0.1396 | max_backward: 1.6794 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.7005 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0532 | Memory: rss_MB: 6542.4 | uss_MB: 6233.1 | child_rss_MB: 3863.9 | allocated_MB: 444.5 | reserved_MB: 26508.0 | inactive_split_MB: 1442.5 +rank7 [2026-10-29 08:48:25,834] INFO : Step 25000/50000 rank7 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4650 | Train Loss Components: Auxiliary: 0.1962 | Cardinality: 0.1385 | Classification: 0.0737 | Classification_binary: 0.1551 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0015 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0546 | Regression_eta: 0.0487 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0730 | LR: 5.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9522s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 2.1073 | max_optimizer_step: 0.2301 | max_tensorboard_logging: 0.7960 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0706 | Memory: rss_MB: 6557.4 | uss_MB: 6248.2 | child_rss_MB: 3873.2 | allocated_MB: 450.1 | reserved_MB: 28588.0 | inactive_split_MB: 1158.9 +rank7 [2026-10-29 08:48:26,771] INFO : Running validation on rank7 +rank7 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 08:48:26,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 08:49:20,640] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:49:21,192] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 08:49:51,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 08:49:53,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 08:49:54,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 08:49:55,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 08:50:05,779] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 08:50:06,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:50:06,729] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 08:50:22,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 08:50:23,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 08:50:24,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 08:50:25,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 08:50:35,954] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:50:36,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:50:36,989] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 08:50:52,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 08:50:53,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 08:50:54,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 08:50:55,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_25000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 08:51:05,773] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 08:56:18,537] INFO : Step 25100/50000 rank7 | Train Loss: 1.0294 | Train Loss Unweighted: 0.4999 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1287 | Classification: 0.0805 | Classification_binary: 0.1838 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1152 | Regression_eta: 0.0016 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0585 | Regression_eta: 0.0291 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0591 | LR: 5.47e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3783s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0070 | max_forward: 0.8414 | max_loss: 0.1600 | max_backward: 1.8703 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 1.1904 | max_tensorboard_flush: 0.0066 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7713.5 | uss_MB: 7075.9 | child_rss_MB: 3865.5 | allocated_MB: 451.4 | reserved_MB: 24128.0 | inactive_split_MB: 997.7 +rank7 [2026-10-29 09:01:01,590] INFO : Step 25200/50000 rank7 | Train Loss: 0.9613 | Train Loss Unweighted: 0.4483 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1300 | Classification: 0.0685 | Classification_binary: 0.1546 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1113 | Regression_eta: 0.0015 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0595 | Regression_eta: 0.0503 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0924 | LR: 5.44e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8755s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0501 | max_forward: 0.8154 | max_loss: 0.1475 | max_backward: 1.8384 | max_optimizer_step: 0.2414 | max_tensorboard_logging: 0.9127 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0649 | Memory: rss_MB: 7715.4 | uss_MB: 7407.3 | child_rss_MB: 3867.4 | allocated_MB: 444.6 | reserved_MB: 24222.0 | inactive_split_MB: 1004.2 +rank7 [2026-10-29 09:05:50,664] INFO : Step 25300/50000 rank7 | Train Loss: 0.8950 | Train Loss Unweighted: 0.4464 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.0917 | Classification: 0.0741 | Classification_binary: 0.1373 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0660 | Regression_eta: 0.0346 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0566 | LR: 5.42e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.9763s | Phase Times: max_pre_step_barrier: 0.1078 | max_batch_to_device: 0.0061 | max_forward: 0.8051 | max_loss: 0.2394 | max_backward: 1.9010 | max_optimizer_step: 0.2997 | max_tensorboard_logging: 0.8676 | max_tensorboard_flush: 0.0127 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 6626.0 | uss_MB: 6007.7 | child_rss_MB: 3856.1 | allocated_MB: 446.4 | reserved_MB: 26164.0 | inactive_split_MB: 1167.8 +rank7 [2026-10-29 09:10:30,565] INFO : Step 25400/50000 rank7 | Train Loss: 1.0696 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1287 | Classification: 0.0880 | Classification_binary: 0.1842 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1174 | Regression_eta: 0.0018 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0539 | Regression_eta: 0.0625 | Regression_pt: 0.0560 | Regression_sin_phi: 0.1133 | LR: 5.39e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.1687s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0070 | max_forward: 0.8456 | max_loss: 0.1484 | max_backward: 2.1955 | max_optimizer_step: 0.2282 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 6630.4 | uss_MB: 6325.0 | child_rss_MB: 3844.7 | allocated_MB: 450.2 | reserved_MB: 26164.0 | inactive_split_MB: 1010.4 +rank7 [2026-10-29 09:15:11,237] INFO : Step 25500/50000 rank7 | Train Loss: 1.0075 | Train Loss Unweighted: 0.4822 | Train Loss Components: Auxiliary: 0.2032 | Cardinality: 0.1308 | Classification: 0.0725 | Classification_binary: 0.1707 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1166 | Regression_eta: 0.0015 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0530 | Regression_eta: 0.0554 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0572 | LR: 5.36e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4685s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0058 | max_forward: 0.7755 | max_loss: 0.1565 | max_backward: 1.7427 | max_optimizer_step: 0.2738 | max_tensorboard_logging: 0.6177 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6624.2 | uss_MB: 6318.1 | child_rss_MB: 3855.6 | allocated_MB: 445.0 | reserved_MB: 28216.0 | inactive_split_MB: 1006.9 +rank7 [2026-10-29 09:19:56,597] INFO : Step 25600/50000 rank7 | Train Loss: 0.9303 | Train Loss Unweighted: 0.4790 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.0975 | Classification: 0.0835 | Classification_binary: 0.1371 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1274 | Regression_eta: 0.0015 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0601 | Regression_eta: 0.0277 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0334 | LR: 5.33e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.9352s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.9925 | max_loss: 0.3282 | max_backward: 2.6243 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 1.0139 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 6621.2 | uss_MB: 6318.3 | child_rss_MB: 3861.9 | allocated_MB: 444.3 | reserved_MB: 29362.0 | inactive_split_MB: 1199.7 +rank7 [2026-10-29 09:24:33,864] INFO : Step 25700/50000 rank7 | Train Loss: 1.0213 | Train Loss Unweighted: 0.5025 | Train Loss Components: Auxiliary: 0.2128 | Cardinality: 0.1221 | Classification: 0.0819 | Classification_binary: 0.1751 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0015 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0583 | Regression_eta: 0.0446 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0649 | LR: 5.30e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2126s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.8356 | max_loss: 0.1490 | max_backward: 2.0918 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 1.0825 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0521 | Memory: rss_MB: 6615.0 | uss_MB: 6314.0 | child_rss_MB: 3839.9 | allocated_MB: 451.5 | reserved_MB: 29362.0 | inactive_split_MB: 1190.5 +rank7 [2026-10-29 09:29:27,624] INFO : Step 25800/50000 rank7 | Train Loss: 1.1284 | Train Loss Unweighted: 0.5663 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1437 | Classification: 0.0929 | Classification_binary: 0.2031 | HitGrouping: 0.1601 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1323 | Regression_eta: 0.0016 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1917 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0593 | Regression_eta: 0.0386 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0711 | LR: 5.27e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1669s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8929 | max_loss: 0.1830 | max_backward: 2.0540 | max_optimizer_step: 0.2316 | max_tensorboard_logging: 0.9419 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6619.3 | uss_MB: 6321.1 | child_rss_MB: 3841.2 | allocated_MB: 449.2 | reserved_MB: 29362.0 | inactive_split_MB: 999.7 +rank7 [2026-10-29 09:34:19,332] INFO : Step 25900/50000 rank7 | Train Loss: 0.9498 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1881 | Cardinality: 0.1142 | Classification: 0.0664 | Classification_binary: 0.1519 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1128 | Regression_eta: 0.0015 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0628 | Regression_eta: 0.0490 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0614 | LR: 5.25e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.7722s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0105 | max_forward: 0.9056 | max_loss: 0.1856 | max_backward: 1.7388 | max_optimizer_step: 0.3189 | max_tensorboard_logging: 0.8212 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6615.0 | uss_MB: 6319.5 | child_rss_MB: 3827.4 | allocated_MB: 448.1 | reserved_MB: 29362.0 | inactive_split_MB: 1007.8 +rank7 [2026-10-29 09:39:03,492] INFO : Step 26000/50000 rank7 | Train Loss: 0.9516 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.0983 | Classification: 0.0780 | Classification_binary: 0.1428 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1256 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0629 | Regression_eta: 0.0381 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0577 | LR: 5.22e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7480s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0392 | max_forward: 0.7937 | max_loss: 0.1394 | max_backward: 1.8373 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8932 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0512 | Memory: rss_MB: 6614.8 | uss_MB: 6319.6 | child_rss_MB: 3848.0 | allocated_MB: 446.0 | reserved_MB: 29362.0 | inactive_split_MB: 1172.9 +rank7 [2026-10-29 09:39:04,390] INFO : Running validation on rank7 +rank7 [2026-10-29 09:39:04,390] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:39:04,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:39:55,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:39:55,910] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 09:40:23,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:40:24,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:40:25,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:40:26,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:40:36,382] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 09:40:37,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:40:37,505] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 09:40:54,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:40:55,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:40:56,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:40:57,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:41:07,916] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 09:41:08,937] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:44:09,566] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 09:44:24,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:44:25,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:44:26,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:44:27,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_26000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:44:37,450] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 09:50:29,555] INFO : Step 26100/50000 rank7 | Train Loss: 1.2021 | Train Loss Unweighted: 0.5257 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.2142 | Classification: 0.0873 | Classification_binary: 0.1946 | HitGrouping: 0.1611 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1194 | Regression_eta: 0.0018 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0633 | Regression_eta: 0.0456 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0563 | LR: 5.19e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.1787s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0072 | max_forward: 0.9192 | max_loss: 0.2532 | max_backward: 1.9966 | max_optimizer_step: 0.2613 | max_tensorboard_logging: 0.9545 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0932 | Memory: rss_MB: 7937.2 | uss_MB: 7632.0 | child_rss_MB: 3818.0 | allocated_MB: 447.0 | reserved_MB: 28008.0 | inactive_split_MB: 969.7 +rank7 [2026-10-29 09:55:15,273] INFO : Step 26200/50000 rank7 | Train Loss: 0.9738 | Train Loss Unweighted: 0.4637 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1474 | Classification: 0.0643 | Classification_binary: 0.1511 | HitGrouping: 0.1583 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1228 | Regression_eta: 0.0012 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0759 | Regression_eta: 0.0320 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0568 | LR: 5.16e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.8150s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0062 | max_forward: 0.9417 | max_loss: 0.1573 | max_backward: 1.9931 | max_optimizer_step: 0.2365 | max_tensorboard_logging: 0.5024 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0583 | Memory: rss_MB: 7937.4 | uss_MB: 7632.0 | child_rss_MB: 3842.4 | allocated_MB: 446.1 | reserved_MB: 29976.0 | inactive_split_MB: 1195.3 +rank7 [2026-10-29 09:59:57,744] INFO : Step 26300/50000 rank7 | Train Loss: 0.9637 | Train Loss Unweighted: 0.4588 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1314 | Classification: 0.0719 | Classification_binary: 0.1405 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1213 | Regression_eta: 0.0014 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0719 | Regression_eta: 0.0336 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0530 | LR: 5.13e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.9320s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0079 | max_forward: 0.7649 | max_loss: 0.1403 | max_backward: 1.9436 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.9853 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6706.7 | uss_MB: 6399.4 | child_rss_MB: 3817.0 | allocated_MB: 447.1 | reserved_MB: 29976.0 | inactive_split_MB: 1212.0 +rank7 [2026-10-29 10:04:36,480] INFO : Step 26400/50000 rank7 | Train Loss: 0.9779 | Train Loss Unweighted: 0.4820 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1052 | Classification: 0.0923 | Classification_binary: 0.1677 | HitGrouping: 0.1509 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1086 | Regression_eta: 0.0016 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0554 | Regression_eta: 0.0429 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0787 | LR: 5.10e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.1494s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.8220 | max_loss: 0.1501 | max_backward: 2.0954 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6706.7 | uss_MB: 5934.4 | child_rss_MB: 3820.6 | allocated_MB: 450.4 | reserved_MB: 32064.0 | inactive_split_MB: 1008.5 +rank7 [2026-10-29 10:09:23,739] INFO : Step 26500/50000 rank7 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4629 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1513 | Classification: 0.0589 | Classification_binary: 0.1781 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1110 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0210 | Classification_binary: 0.1568 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0560 | Regression_eta: 0.0302 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0624 | LR: 5.08e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0938s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.8016 | max_loss: 0.5291 | max_backward: 2.0682 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8092 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0534 | Memory: rss_MB: 6705.0 | uss_MB: 6400.5 | child_rss_MB: 3823.9 | allocated_MB: 449.5 | reserved_MB: 33310.0 | inactive_split_MB: 973.8 +rank7 [2026-10-29 10:14:05,781] INFO : Step 26600/50000 rank7 | Train Loss: 0.8946 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.1780 | Cardinality: 0.1018 | Classification: 0.0771 | Classification_binary: 0.1269 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1204 | Regression_eta: 0.0014 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0561 | Regression_eta: 0.0444 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0554 | LR: 5.05e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6619s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7729 | max_loss: 0.5059 | max_backward: 1.7560 | max_optimizer_step: 0.2330 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 6702.4 | uss_MB: 6397.6 | child_rss_MB: 3850.2 | allocated_MB: 446.5 | reserved_MB: 33310.0 | inactive_split_MB: 1155.4 +rank7 [2026-10-29 10:19:02,168] INFO : Step 26700/50000 rank7 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4925 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1290 | Classification: 0.0904 | Classification_binary: 0.1599 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0303 | Classification_binary: 0.2155 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0590 | Regression_eta: 0.0335 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0582 | LR: 5.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2155s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0482 | max_forward: 0.8321 | max_loss: 0.1442 | max_backward: 1.8490 | max_optimizer_step: 0.2336 | max_tensorboard_logging: 1.2936 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6695.0 | uss_MB: 6392.3 | child_rss_MB: 3823.2 | allocated_MB: 449.1 | reserved_MB: 33310.0 | inactive_split_MB: 1390.9 +rank7 [2026-10-29 10:23:58,674] INFO : Step 26800/50000 rank7 | Train Loss: 1.0510 | Train Loss Unweighted: 0.5118 | Train Loss Components: Auxiliary: 0.2171 | Cardinality: 0.1324 | Classification: 0.0818 | Classification_binary: 0.1931 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1162 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0708 | Regression_eta: 0.0418 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0697 | LR: 4.99e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.5314s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0077 | max_forward: 0.8167 | max_loss: 0.1490 | max_backward: 1.8226 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 6701.0 | uss_MB: 6397.4 | child_rss_MB: 3829.2 | allocated_MB: 449.6 | reserved_MB: 33310.0 | inactive_split_MB: 1390.4 +rank7 [2026-10-29 10:28:51,386] INFO : Step 26900/50000 rank7 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4398 | Train Loss Components: Auxiliary: 0.1841 | Cardinality: 0.1066 | Classification: 0.0635 | Classification_binary: 0.1406 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1160 | Regression_eta: 0.0012 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0626 | Regression_eta: 0.0420 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0764 | LR: 4.96e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.5619s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0073 | max_forward: 0.8505 | max_loss: 0.1790 | max_backward: 1.9155 | max_optimizer_step: 0.2409 | max_tensorboard_logging: 0.5116 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0666 | Memory: rss_MB: 6701.9 | uss_MB: 6398.5 | child_rss_MB: 3826.6 | allocated_MB: 449.0 | reserved_MB: 33314.0 | inactive_split_MB: 1395.0 +rank7 [2026-10-29 10:33:37,218] INFO : Step 27000/50000 rank7 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4718 | Train Loss Components: Auxiliary: 0.1982 | Cardinality: 0.1204 | Classification: 0.0871 | Classification_binary: 0.1438 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0490 | Regression_eta: 0.0433 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0555 | LR: 4.94e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.6931s | Phase Times: max_pre_step_barrier: 0.0297 | max_batch_to_device: 0.0068 | max_forward: 1.1664 | max_loss: 0.2275 | max_backward: 2.5026 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5391 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6702.6 | uss_MB: 6240.2 | child_rss_MB: 3856.9 | allocated_MB: 448.6 | reserved_MB: 33314.0 | inactive_split_MB: 973.9 +rank7 [2026-10-29 10:33:38,215] INFO : Running validation on rank7 +rank7 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 10:33:38,215] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 10:34:33,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:34:34,221] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 10:35:00,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 10:35:01,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 10:35:02,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 10:35:03,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 10:35:13,338] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:35:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:35:14,236] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 10:35:30,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 10:35:31,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 10:35:32,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 10:35:33,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 10:35:43,514] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:35:44,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:35:45,038] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 10:36:00,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 10:36:01,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 10:36:02,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 10:36:03,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_27000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 10:36:13,306] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 10:41:28,285] INFO : Step 27100/50000 rank7 | Train Loss: 0.9994 | Train Loss Unweighted: 0.5011 | Train Loss Components: Auxiliary: 0.2062 | Cardinality: 0.0863 | Classification: 0.0877 | Classification_binary: 0.1814 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1138 | Regression_eta: 0.0016 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0532 | Regression_eta: 0.0522 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0717 | LR: 4.91e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0069 | max_forward: 0.8465 | max_loss: 0.1485 | max_backward: 2.0441 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 7905.1 | uss_MB: 7599.9 | child_rss_MB: 3873.6 | allocated_MB: 448.6 | reserved_MB: 26258.0 | inactive_split_MB: 1209.4 +rank7 [2026-10-29 10:46:22,294] INFO : Step 27200/50000 rank7 | Train Loss: 1.0064 | Train Loss Unweighted: 0.5171 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.1033 | Classification: 0.0696 | Classification_binary: 0.1685 | HitGrouping: 0.1623 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1369 | Regression_eta: 0.0017 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0283 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0672 | Regression_eta: 0.0423 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0619 | LR: 4.88e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8256s | Phase Times: max_pre_step_barrier: 0.1104 | max_batch_to_device: 0.0195 | max_forward: 1.0804 | max_loss: 0.1530 | max_backward: 1.7911 | max_optimizer_step: 0.2901 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7903.2 | uss_MB: 7597.5 | child_rss_MB: 3876.9 | allocated_MB: 447.7 | reserved_MB: 27994.0 | inactive_split_MB: 937.8 +rank7 [2026-10-29 10:51:33,265] INFO : Step 27300/50000 rank7 | Train Loss: 0.8458 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0982 | Classification: 0.0709 | Classification_binary: 0.1229 | HitGrouping: 0.1528 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1078 | Regression_eta: 0.0012 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0549 | Regression_eta: 0.0317 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0667 | LR: 4.85e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.6694s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7829 | max_loss: 0.2377 | max_backward: 1.7514 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8639 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6771.4 | uss_MB: 6463.7 | child_rss_MB: 3870.4 | allocated_MB: 447.2 | reserved_MB: 27994.0 | inactive_split_MB: 1394.9 +rank7 [2026-10-29 10:56:31,405] INFO : Step 27400/50000 rank7 | Train Loss: 1.0514 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2207 | Cardinality: 0.1299 | Classification: 0.0900 | Classification_binary: 0.1800 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1179 | Regression_eta: 0.0015 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1943 | Regression_cos_phi: 0.1390 | Regression_energy: 0.0650 | Regression_eta: 0.0639 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0934 | LR: 4.82e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.9749s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.9114 | max_loss: 0.1538 | max_backward: 2.1643 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.5591 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1604 | Memory: rss_MB: 6777.4 | uss_MB: 6469.8 | child_rss_MB: 3870.4 | allocated_MB: 449.2 | reserved_MB: 27994.0 | inactive_split_MB: 947.9 +rank7 [2026-10-29 11:01:35,253] INFO : Step 27500/50000 rank7 | Train Loss: 0.9852 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1189 | Classification: 0.0733 | Classification_binary: 0.1815 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0015 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0541 | Regression_eta: 0.0383 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0666 | LR: 4.80e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.3130s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 1.2060 | max_loss: 0.3022 | max_backward: 2.2587 | max_optimizer_step: 0.2909 | max_tensorboard_logging: 0.7431 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 6763.1 | uss_MB: 6456.6 | child_rss_MB: 3872.7 | allocated_MB: 448.0 | reserved_MB: 29994.0 | inactive_split_MB: 939.7 +rank7 [2026-10-29 11:08:29,071] INFO : Step 27600/50000 rank7 | Train Loss: 0.8726 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1794 | Cardinality: 0.1047 | Classification: 0.0801 | Classification_binary: 0.1401 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1037 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0506 | Regression_eta: 0.0234 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0492 | LR: 4.77e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.3322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1141 | max_forward: 1.1031 | max_loss: 0.3750 | max_backward: 3.2301 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.7828 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6766.6 | uss_MB: 6462.9 | child_rss_MB: 3873.5 | allocated_MB: 449.0 | reserved_MB: 29994.0 | inactive_split_MB: 1196.8 +rank7 [2026-10-29 11:13:33,057] INFO : Step 27700/50000 rank7 | Train Loss: 0.8901 | Train Loss Unweighted: 0.4487 | Train Loss Components: Auxiliary: 0.1831 | Cardinality: 0.1045 | Classification: 0.0733 | Classification_binary: 0.1411 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0013 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0615 | Regression_eta: 0.0466 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0447 | LR: 4.74e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7962s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8196 | max_loss: 0.1716 | max_backward: 2.0252 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.7203 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 6770.9 | uss_MB: 6466.3 | child_rss_MB: 3873.9 | allocated_MB: 449.5 | reserved_MB: 29994.0 | inactive_split_MB: 1196.5 +rank7 [2026-10-29 11:18:30,826] INFO : Step 27800/50000 rank7 | Train Loss: 1.0203 | Train Loss Unweighted: 0.4823 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1323 | Classification: 0.0748 | Classification_binary: 0.1905 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0017 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.2107 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0602 | Regression_eta: 0.0636 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0618 | LR: 4.71e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0110s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.8463 | max_loss: 0.1533 | max_backward: 2.0151 | max_optimizer_step: 0.2332 | max_tensorboard_logging: 0.8667 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 6767.9 | uss_MB: 6463.7 | child_rss_MB: 3866.4 | allocated_MB: 450.6 | reserved_MB: 29994.0 | inactive_split_MB: 1204.1 +rank7 [2026-10-29 11:23:38,363] INFO : Step 27900/50000 rank7 | Train Loss: 0.9847 | Train Loss Unweighted: 0.4804 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1243 | Classification: 0.0778 | Classification_binary: 0.1608 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1193 | Regression_eta: 0.0015 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0522 | Regression_eta: 0.0505 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0495 | LR: 4.68e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.8884s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.7715 | max_loss: 0.1639 | max_backward: 1.7363 | max_optimizer_step: 0.3208 | max_tensorboard_logging: 0.8601 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 6768.1 | uss_MB: 6463.3 | child_rss_MB: 3870.1 | allocated_MB: 446.0 | reserved_MB: 29994.0 | inactive_split_MB: 1005.1 +rank7 [2026-10-29 11:28:54,871] INFO : Step 28000/50000 rank7 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1017 | Classification: 0.0731 | Classification_binary: 0.1312 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1107 | Regression_eta: 0.0015 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0476 | Regression_eta: 0.0342 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0826 | LR: 4.66e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.9074s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0062 | max_forward: 0.8637 | max_loss: 0.1412 | max_backward: 2.2872 | max_optimizer_step: 0.2602 | max_tensorboard_logging: 0.9926 | max_tensorboard_flush: 0.0029 | max_post_step_barrier: 0.0582 | Memory: rss_MB: 6768.6 | uss_MB: 6170.8 | child_rss_MB: 3862.0 | allocated_MB: 447.8 | reserved_MB: 29994.0 | inactive_split_MB: 1001.1 +rank7 [2026-10-29 11:28:55,940] INFO : Running validation on rank7 +rank7 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 11:28:55,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 11:29:47,986] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:29:48,519] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 11:30:14,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 11:30:15,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 11:30:16,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 11:30:17,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 11:30:27,436] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 11:30:28,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:30:28,728] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 11:30:44,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 11:30:45,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 11:30:46,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 11:30:47,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 11:30:57,434] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 11:30:57,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:30:58,464] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 11:31:13,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 11:31:14,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 11:31:15,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 11:31:16,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_28000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 11:31:26,212] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 11:37:01,722] INFO : Step 28100/50000 rank7 | Train Loss: 0.9661 | Train Loss Unweighted: 0.4847 | Train Loss Components: Auxiliary: 0.2041 | Cardinality: 0.1018 | Classification: 0.0869 | Classification_binary: 0.1674 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1134 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0709 | Regression_eta: 0.0415 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0632 | LR: 4.63e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.4997s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.1506 | max_backward: 2.0589 | max_optimizer_step: 0.2481 | max_tensorboard_logging: 1.0841 | max_tensorboard_flush: 0.0130 | max_post_step_barrier: 0.0859 | Memory: rss_MB: 7908.3 | uss_MB: 7599.9 | child_rss_MB: 3838.1 | allocated_MB: 447.2 | reserved_MB: 25198.0 | inactive_split_MB: 945.2 +rank7 [2026-10-29 11:42:06,043] INFO : Step 28200/50000 rank7 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1110 | Classification: 0.0708 | Classification_binary: 0.1592 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1083 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0553 | Regression_eta: 0.0444 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0564 | LR: 4.60e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.0742s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 1.0135 | max_loss: 0.2261 | max_backward: 3.1098 | max_optimizer_step: 0.3542 | max_tensorboard_logging: 0.6434 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 7909.8 | uss_MB: 7600.6 | child_rss_MB: 3831.5 | allocated_MB: 447.2 | reserved_MB: 29180.0 | inactive_split_MB: 1285.2 +rank7 [2026-10-29 11:47:09,276] INFO : Step 28300/50000 rank7 | Train Loss: 0.8954 | Train Loss Unweighted: 0.4347 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0963 | Classification: 0.0651 | Classification_binary: 0.1264 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1197 | Regression_eta: 0.0017 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1039 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0474 | Regression_eta: 0.0480 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0796 | LR: 4.57e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.0923s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.8314 | max_loss: 0.2144 | max_backward: 1.9437 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 0.6472 | max_tensorboard_flush: 0.0151 | max_post_step_barrier: 0.0591 | Memory: rss_MB: 6868.8 | uss_MB: 6019.1 | child_rss_MB: 3837.3 | allocated_MB: 446.2 | reserved_MB: 29180.0 | inactive_split_MB: 1273.3 +rank7 [2026-10-29 11:52:05,668] INFO : Step 28400/50000 rank7 | Train Loss: 1.8511 | Train Loss Unweighted: 1.0454 | Train Loss Components: Auxiliary: 0.3958 | Cardinality: 0.3225 | Classification: 0.2714 | Classification_binary: 0.2490 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2612 | Regression_eta: 0.0026 | Regression_pt: 0.2589 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.1761 | Classification_binary: 0.2545 | Regression_cos_phi: 0.0679 | Regression_energy: 0.1261 | Regression_eta: 0.0716 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0969 | LR: 4.55e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0054 | max_batch_to_device: 0.0068 | max_forward: 0.8384 | max_loss: 0.1482 | max_backward: 1.9822 | max_optimizer_step: 0.2377 | max_tensorboard_logging: 0.5361 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 6866.5 | uss_MB: 6559.8 | child_rss_MB: 3830.3 | allocated_MB: 449.4 | reserved_MB: 29180.0 | inactive_split_MB: 1079.9 +rank7 [2026-10-29 11:56:56,840] INFO : Step 28500/50000 rank7 | Train Loss: 0.9993 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2064 | Cardinality: 0.1236 | Classification: 0.0780 | Classification_binary: 0.1811 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1090 | Regression_eta: 0.0016 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0554 | Regression_eta: 0.0389 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0529 | LR: 4.52e-04 | DataLoad Time: 0.0128s | Model Forward Time: 4.9932s | Phase Times: max_pre_step_barrier: 0.0078 | max_batch_to_device: 0.0645 | max_forward: 0.8865 | max_loss: 0.1557 | max_backward: 1.9315 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.7655 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0653 | Memory: rss_MB: 6858.2 | uss_MB: 6159.4 | child_rss_MB: 3830.8 | allocated_MB: 447.7 | reserved_MB: 31198.0 | inactive_split_MB: 984.3 +rank7 [2026-10-29 12:01:46,934] INFO : Step 28600/50000 rank7 | Train Loss: 0.9109 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1095 | Classification: 0.0764 | Classification_binary: 0.1417 | HitGrouping: 0.1561 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1102 | Regression_eta: 0.0013 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0627 | Regression_eta: 0.0411 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0891 | LR: 4.49e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7856 | max_loss: 0.1468 | max_backward: 1.8858 | max_optimizer_step: 0.2397 | max_tensorboard_logging: 0.5278 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0573 | Memory: rss_MB: 6864.2 | uss_MB: 6558.8 | child_rss_MB: 3832.1 | allocated_MB: 447.5 | reserved_MB: 31200.0 | inactive_split_MB: 1204.1 +rank7 [2026-10-29 12:06:39,995] INFO : Step 28700/50000 rank7 | Train Loss: 0.9015 | Train Loss Unweighted: 0.4429 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.0976 | Classification: 0.0704 | Classification_binary: 0.1442 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1119 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0549 | Regression_eta: 0.0446 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0509 | LR: 4.46e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.5059s | Phase Times: max_pre_step_barrier: 0.0130 | max_batch_to_device: 0.0063 | max_forward: 1.0462 | max_loss: 0.2228 | max_backward: 2.4186 | max_optimizer_step: 0.2374 | max_tensorboard_logging: 0.5874 | max_tensorboard_flush: 0.0132 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6864.2 | uss_MB: 6530.8 | child_rss_MB: 3831.0 | allocated_MB: 448.1 | reserved_MB: 31200.0 | inactive_split_MB: 1427.7 +rank7 [2026-10-29 12:11:31,023] INFO : Step 28800/50000 rank7 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4694 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1157 | Classification: 0.0783 | Classification_binary: 0.1657 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0509 | Regression_eta: 0.0491 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0678 | LR: 4.44e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.9313s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8578 | max_loss: 0.2125 | max_backward: 2.5170 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 1.0201 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 6860.9 | uss_MB: 6304.5 | child_rss_MB: 3822.4 | allocated_MB: 449.7 | reserved_MB: 31200.0 | inactive_split_MB: 1430.1 +rank7 [2026-10-29 12:16:17,097] INFO : Step 28900/50000 rank7 | Train Loss: 0.9062 | Train Loss Unweighted: 0.4330 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1174 | Classification: 0.0731 | Classification_binary: 0.1471 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1048 | Regression_eta: 0.0015 | Regression_pt: 0.1051 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0531 | Regression_eta: 0.0303 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0556 | LR: 4.41e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7005s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7955 | max_loss: 0.1506 | max_backward: 1.7809 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.8257 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 6861.9 | uss_MB: 6558.6 | child_rss_MB: 3841.1 | allocated_MB: 450.1 | reserved_MB: 31200.0 | inactive_split_MB: 1289.3 +rank7 [2026-10-29 12:21:10,134] INFO : Step 29000/50000 rank7 | Train Loss: 0.8982 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1056 | Classification: 0.0740 | Classification_binary: 0.1306 | HitGrouping: 0.1541 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1143 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0518 | Regression_eta: 0.0407 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0760 | LR: 4.38e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9918s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8296 | max_loss: 0.1434 | max_backward: 1.9190 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0128 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 6858.1 | uss_MB: 4808.7 | child_rss_MB: 3841.0 | allocated_MB: 453.4 | reserved_MB: 31200.0 | inactive_split_MB: 1121.7 +rank7 [2026-10-29 12:21:11,232] INFO : Running validation on rank7 +rank7 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 12:21:11,232] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 12:22:04,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:22:04,810] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 12:22:31,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 12:22:32,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 12:22:33,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 12:22:34,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 12:22:44,961] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 12:22:45,452] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:22:45,974] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 12:23:02,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 12:23:03,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 12:23:04,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 12:23:05,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 12:23:15,121] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 12:23:15,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:23:16,443] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 12:23:32,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 12:23:33,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 12:23:33,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 12:23:34,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_29000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 12:23:44,646] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 12:28:57,786] INFO : Step 29100/50000 rank7 | Train Loss: 0.9929 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2052 | Cardinality: 0.1224 | Classification: 0.0735 | Classification_binary: 0.1688 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0561 | Regression_eta: 0.0329 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0442 | LR: 4.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1720s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.8393 | max_loss: 0.5591 | max_backward: 2.0367 | max_optimizer_step: 0.2914 | max_tensorboard_logging: 0.8317 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0740 | Memory: rss_MB: 7938.9 | uss_MB: 7634.8 | child_rss_MB: 3818.0 | allocated_MB: 448.1 | reserved_MB: 26822.0 | inactive_split_MB: 1175.9 +rank7 [2026-10-29 12:33:43,192] INFO : Step 29200/50000 rank7 | Train Loss: 0.9980 | Train Loss Unweighted: 0.4508 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1371 | Classification: 0.0736 | Classification_binary: 0.1638 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1046 | Regression_eta: 0.0018 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0555 | Regression_eta: 0.0486 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0650 | LR: 4.33e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5642s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.9135 | max_loss: 0.1559 | max_backward: 1.8579 | max_optimizer_step: 0.2553 | max_tensorboard_logging: 0.5052 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 7940.1 | uss_MB: 7638.2 | child_rss_MB: 3835.9 | allocated_MB: 451.6 | reserved_MB: 28864.0 | inactive_split_MB: 959.8 +rank7 [2026-10-29 14:49:26,355] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-29 14:49:26,388] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-29 14:49:54,070] INFO : Instantiating model +rank7 [2026-10-29 14:49:54,071] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-29 14:49:54,071] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-29 14:49:54,089] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-29 14:49:54,089] INFO : nn0 parameters: 171776 +rank7 [2026-10-29 14:49:54,089] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-29 14:49:54,089] INFO : Initializing backbone layer 0 +rank7 [2026-10-29 14:49:54,127] INFO : Initializing backbone layer 1 +rank7 [2026-10-29 14:49:54,141] INFO : Initializing backbone layer 2 +rank7 [2026-10-29 14:49:54,156] INFO : Initializing backbone layer 3 +rank7 [2026-10-29 14:49:54,170] INFO : Initializing backbone layer 4 +rank7 [2026-10-29 14:49:54,184] INFO : Initializing backbone layer 5 +rank7 [2026-10-29 14:49:54,194] INFO : Backbone initialization took 0.11s +rank7 [2026-10-29 14:49:54,195] INFO : backbone parameters: 7901376 +rank7 [2026-10-29 14:49:54,195] INFO : Initializing output DNNs +rank7 [2026-10-29 14:49:54,229] INFO : Output DNNs initialization took 0.03s +rank7 [2026-10-29 14:49:54,229] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-29 14:49:54,229] INFO : jagged_attention enabled=False +rank7 [2026-10-29 14:49:54,229] INFO : flash_attn_varlen enabled=False +rank7 [2026-10-29 14:49:54,229] INFO : task_queries enabled=False +rank7 [2026-10-29 14:49:54,229] INFO : classification_norm parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : regression_norm parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : classification_query parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : regression_query parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : classification_readout parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : regression_readout parameters: 0 +rank7 [2026-10-29 14:49:54,229] INFO : output_mode=set +rank7 [2026-10-29 14:49:54,230] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-29 14:49:54,230] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_pid parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_pt parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_eta parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-29 14:49:54,230] INFO : nn_energy parameters: 0 +rank7 [2026-10-29 14:49:54,231] INFO : Total MLPF parameters: 12786964 +rank7 [2026-10-29 14:49:54,231] INFO : MLPF __init__ done +rank7 [2026-10-29 14:49:54,231] INFO : Instantiated model +rank7 [2026-10-29 14:49:54,231] INFO : Moving model to device rank=7 +rank7 [2026-10-29 14:49:54,250] INFO : Moved model to device rank=7 +rank7 [2026-10-29 14:49:54,252] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-29 14:49:54,791] INFO : Restoring training from step 29001 +rank7 [2026-10-29 14:49:54,799] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/checkpoints/checkpoint-29000.pth +rank7 [2026-10-29 14:49:54,799] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-29 14:49:55,362] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-29 14:49:55,512] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank7 [2026-10-29 14:49:55,512] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-29 14:49:55,514] INFO : Trainable parameters: 12786964 +rank7 [2026-10-29 14:49:55,514] INFO : Non-trainable parameters: 23268 +rank7 [2026-10-29 14:49:55,514] INFO : Total parameters: 12810232 +rank7 [2026-10-29 14:49:55,517] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-29 14:49:55,517] INFO : Getting dataloaders +rank7 [2026-10-29 14:49:55,517] INFO : sample=clic_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:01,361] INFO : sample=clic_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:02,346] INFO : sample=clic_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:03,544] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-29 14:50:03,545] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-29 14:50:03,545] INFO : sample=clic_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:03,813] INFO : sample=clic_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:04,081] INFO : sample=clic_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:50:04,333] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-29 14:50:04,334] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-29 14:50:18,124] INFO : Got dataloaders +rank7 [2026-10-29 14:50:18,124] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [7892] of epochs [1] +rank7 [2026-10-29 14:50:18,125] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank7 [2026-10-29 14:50:18,125] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank7 [2026-10-29 14:50:18,125] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank7 [2026-10-29 14:50:19,535] INFO : loader split=train rank=7 len=21108 +rank7 [2026-10-29 14:50:19,535] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-29 14:50:19,536] INFO : Starting training from step 29001 to 50000 on rank 7 of 8 +rank7 [2026-10-29 14:50:19,536] INFO : Creating train_iterator +rank7 [2026-10-29 14:50:19,536] INFO : ResumableSampler physical:train loading state: start_index=126272 +rank7 [2026-10-29 14:50:33,359] INFO : Created train_iterator +rank7 [2026-10-29 14:55:12,919] INFO : Step 29100/50000 rank7 | Train Loss: 0.9715 | Train Loss Unweighted: 0.4755 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1146 | Classification: 0.0785 | Classification_binary: 0.1708 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1112 | Regression_eta: 0.0014 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0625 | Regression_eta: 0.0381 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0507 | LR: 4.35e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7157s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7099 | max_loss: 0.1488 | max_backward: 1.6117 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 4767.7 | uss_MB: 3469.7 | child_rss_MB: 6385.8 | allocated_MB: 498.4 | reserved_MB: 27564.0 | inactive_split_MB: 1008.2 +rank7 [2026-10-29 14:59:46,788] INFO : Step 29200/50000 rank7 | Train Loss: 1.0116 | Train Loss Unweighted: 0.4479 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1437 | Classification: 0.0741 | Classification_binary: 0.1666 | HitGrouping: 0.1546 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1019 | Regression_eta: 0.0019 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0513 | Regression_eta: 0.0434 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0556 | LR: 4.33e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3628s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8353 | max_loss: 0.2530 | max_backward: 2.1290 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.4385 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4767.6 | uss_MB: 3473.5 | child_rss_MB: 6386.9 | allocated_MB: 500.2 | reserved_MB: 29606.0 | inactive_split_MB: 870.1 +rank7 [2026-10-29 15:04:21,833] INFO : Step 29300/50000 rank7 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4198 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.0947 | Classification: 0.0643 | Classification_binary: 0.1307 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0011 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0602 | Regression_eta: 0.0370 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0665 | LR: 4.30e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.0896s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7216 | max_loss: 0.1464 | max_backward: 1.6809 | max_optimizer_step: 0.1573 | max_tensorboard_logging: 0.8131 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0020 | Memory: rss_MB: 4766.7 | uss_MB: 3473.9 | child_rss_MB: 6379.3 | allocated_MB: 493.7 | reserved_MB: 29606.0 | inactive_split_MB: 1217.6 +rank7 [2026-10-29 15:08:57,030] INFO : Step 29400/50000 rank7 | Train Loss: 0.9170 | Train Loss Unweighted: 0.4495 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1024 | Classification: 0.0793 | Classification_binary: 0.1502 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1079 | Regression_eta: 0.0015 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0521 | Regression_eta: 0.0384 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0754 | LR: 4.27e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2718s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8784 | max_loss: 0.1429 | max_backward: 2.0163 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.5617 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4768.0 | uss_MB: 3478.8 | child_rss_MB: 6386.0 | allocated_MB: 498.3 | reserved_MB: 29606.0 | inactive_split_MB: 1208.3 +rank7 [2026-10-29 15:13:31,010] INFO : Step 29500/50000 rank7 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1060 | Classification: 0.0774 | Classification_binary: 0.1808 | HitGrouping: 0.1555 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1100 | Regression_eta: 0.0015 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.2013 | Regression_cos_phi: 0.1321 | Regression_energy: 0.0506 | Regression_eta: 0.0479 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0522 | LR: 4.24e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4978s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.7891 | max_loss: 0.2177 | max_backward: 1.8587 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.9013 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4766.9 | uss_MB: 3478.6 | child_rss_MB: 6368.9 | allocated_MB: 496.9 | reserved_MB: 31714.0 | inactive_split_MB: 869.9 +rank7 [2026-10-29 15:18:11,477] INFO : Step 29600/50000 rank7 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4356 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0988 | Classification: 0.0630 | Classification_binary: 0.1351 | HitGrouping: 0.1560 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0557 | Regression_eta: 0.0293 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0386 | LR: 4.22e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5920s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0431 | max_forward: 0.7686 | max_loss: 0.1473 | max_backward: 1.8386 | max_optimizer_step: 0.1616 | max_tensorboard_logging: 1.1152 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4773.9 | uss_MB: 3478.6 | child_rss_MB: 6383.3 | allocated_MB: 496.2 | reserved_MB: 31714.0 | inactive_split_MB: 1206.0 +rank7 [2026-10-29 15:22:45,626] INFO : Step 29700/50000 rank7 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4233 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1064 | Classification: 0.0749 | Classification_binary: 0.1368 | HitGrouping: 0.1540 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0448 | Regression_eta: 0.0291 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0316 | LR: 4.19e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6721s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1039 | max_forward: 0.7980 | max_loss: 0.1479 | max_backward: 2.0904 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.9498 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4773.8 | uss_MB: 3480.2 | child_rss_MB: 6391.4 | allocated_MB: 500.2 | reserved_MB: 31714.0 | inactive_split_MB: 871.8 +rank7 [2026-10-29 15:27:18,562] INFO : Step 29800/50000 rank7 | Train Loss: 1.0328 | Train Loss Unweighted: 0.4916 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1315 | Classification: 0.0814 | Classification_binary: 0.1841 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1106 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0518 | Regression_eta: 0.0392 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0506 | LR: 4.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6301s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0083 | max_forward: 0.8088 | max_loss: 0.2011 | max_backward: 1.9671 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 0.9973 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4742.9 | uss_MB: 3479.7 | child_rss_MB: 6232.3 | allocated_MB: 501.4 | reserved_MB: 31714.0 | inactive_split_MB: 883.5 +rank7 [2026-10-29 15:31:51,911] INFO : Step 29900/50000 rank7 | Train Loss: 0.9815 | Train Loss Unweighted: 0.4662 | Train Loss Components: Auxiliary: 0.2034 | Cardinality: 0.1305 | Classification: 0.0689 | Classification_binary: 0.1659 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0647 | Regression_eta: 0.0419 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0636 | LR: 4.14e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5842s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0148 | max_forward: 0.7661 | max_loss: 0.1479 | max_backward: 1.8104 | max_optimizer_step: 0.1556 | max_tensorboard_logging: 1.1059 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0018 | Memory: rss_MB: 4715.8 | uss_MB: 3482.6 | child_rss_MB: 6133.2 | allocated_MB: 498.3 | reserved_MB: 31714.0 | inactive_split_MB: 883.9 +rank7 [2026-10-29 15:36:26,787] INFO : Step 30000/50000 rank7 | Train Loss: 0.8609 | Train Loss Unweighted: 0.4144 | Train Loss Components: Auxiliary: 0.1730 | Cardinality: 0.1040 | Classification: 0.0703 | Classification_binary: 0.1342 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1034 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1106 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0514 | Regression_eta: 0.0310 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0620 | LR: 4.11e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4513s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.8515 | max_loss: 0.1484 | max_backward: 1.8688 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.9564 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4675.6 | uss_MB: 3480.7 | child_rss_MB: 6047.2 | allocated_MB: 496.3 | reserved_MB: 31714.0 | inactive_split_MB: 1203.6 +rank7 [2026-10-29 15:36:27,739] INFO : Running validation on rank7 +rank7 [2026-10-29 15:36:27,740] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 15:36:27,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 15:37:19,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:37:19,773] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 15:37:50,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 15:37:51,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 15:37:52,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 15:37:53,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 15:38:03,903] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 15:38:04,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:38:04,946] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 15:38:20,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 15:38:21,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 15:38:22,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 15:38:23,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 15:38:33,780] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 15:38:34,554] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:38:35,047] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 15:38:50,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 15:38:51,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 15:38:51,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 15:38:52,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_30000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 15:39:02,816] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 15:44:19,185] INFO : Step 30100/50000 rank7 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4753 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.1412 | Classification: 0.0787 | Classification_binary: 0.1649 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1136 | Regression_eta: 0.0016 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0633 | Regression_eta: 0.0406 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0488 | LR: 4.08e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7111s | Phase Times: max_pre_step_barrier: 0.0215 | max_batch_to_device: 0.0070 | max_forward: 1.0040 | max_loss: 0.2521 | max_backward: 1.8194 | max_optimizer_step: 0.2900 | max_tensorboard_logging: 0.9348 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 6395.4 | uss_MB: 5200.0 | child_rss_MB: 6060.2 | allocated_MB: 498.7 | reserved_MB: 26168.0 | inactive_split_MB: 1381.6 +rank7 [2026-10-29 15:49:01,373] INFO : Step 30200/50000 rank7 | Train Loss: 0.9804 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2059 | Cardinality: 0.1275 | Classification: 0.0715 | Classification_binary: 0.1721 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1127 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0526 | Regression_eta: 0.0318 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0426 | LR: 4.06e-04 | DataLoad Time: 0.0269s | Model Forward Time: 4.7624s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0059 | max_forward: 1.1706 | max_loss: 0.1878 | max_backward: 2.0942 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0968 | Memory: rss_MB: 6384.8 | uss_MB: 5201.2 | child_rss_MB: 6060.9 | allocated_MB: 498.2 | reserved_MB: 26168.0 | inactive_split_MB: 1430.8 +rank7 [2026-10-29 15:53:38,868] INFO : Step 30300/50000 rank7 | Train Loss: 0.8902 | Train Loss Unweighted: 0.4127 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.1090 | Classification: 0.0715 | Classification_binary: 0.1372 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1005 | Regression_eta: 0.0016 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0519 | Regression_eta: 0.0594 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0584 | LR: 4.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3382s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8769 | max_loss: 0.1405 | max_backward: 2.0469 | max_optimizer_step: 0.1702 | max_tensorboard_logging: 0.5643 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5503.4 | uss_MB: 4324.2 | child_rss_MB: 6058.8 | allocated_MB: 496.5 | reserved_MB: 26170.0 | inactive_split_MB: 1424.1 +rank7 [2026-10-29 15:58:13,388] INFO : Step 30400/50000 rank7 | Train Loss: 0.9274 | Train Loss Unweighted: 0.4558 | Train Loss Components: Auxiliary: 0.1891 | Cardinality: 0.1095 | Classification: 0.0805 | Classification_binary: 0.1477 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1123 | Regression_eta: 0.0014 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0645 | Regression_eta: 0.0312 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0608 | LR: 4.00e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4501s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0303 | max_forward: 0.8603 | max_loss: 0.1694 | max_backward: 2.1698 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.5300 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 5495.1 | uss_MB: 4327.1 | child_rss_MB: 6037.2 | allocated_MB: 497.6 | reserved_MB: 28150.0 | inactive_split_MB: 1185.8 +rank7 [2026-10-29 16:02:51,486] INFO : Step 30500/50000 rank7 | Train Loss: 1.4749 | Train Loss Unweighted: 0.6173 | Train Loss Components: Auxiliary: 0.2854 | Cardinality: 0.3860 | Classification: 0.0960 | Classification_binary: 0.2336 | HitGrouping: 0.1689 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1408 | Regression_eta: 0.0017 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0753 | Regression_eta: 0.0467 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0689 | LR: 3.98e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4561s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0157 | max_forward: 0.8899 | max_loss: 0.1553 | max_backward: 1.9805 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1043 | Memory: rss_MB: 5472.8 | uss_MB: 4307.8 | child_rss_MB: 6018.0 | allocated_MB: 498.2 | reserved_MB: 28158.0 | inactive_split_MB: 1106.9 +rank7 [2026-10-29 16:07:32,049] INFO : Step 30600/50000 rank7 | Train Loss: 0.8813 | Train Loss Unweighted: 0.4317 | Train Loss Components: Auxiliary: 0.1818 | Cardinality: 0.0951 | Classification: 0.0755 | Classification_binary: 0.1408 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0014 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0486 | Regression_eta: 0.0389 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0504 | LR: 3.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9370s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7378 | max_loss: 0.1599 | max_backward: 1.6692 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0127 | Memory: rss_MB: 5460.9 | uss_MB: 4322.8 | child_rss_MB: 5931.3 | allocated_MB: 496.1 | reserved_MB: 28158.0 | inactive_split_MB: 1430.1 +rank7 [2026-10-29 16:12:18,779] INFO : Step 30700/50000 rank7 | Train Loss: 0.8888 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1809 | Cardinality: 0.0928 | Classification: 0.0693 | Classification_binary: 0.1353 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1097 | Regression_eta: 0.0014 | Regression_pt: 0.1109 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0665 | Regression_eta: 0.0552 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0605 | LR: 3.92e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2831s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7578 | max_loss: 0.1494 | max_backward: 1.6025 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0148 | Memory: rss_MB: 5456.7 | uss_MB: 4325.0 | child_rss_MB: 5911.8 | allocated_MB: 498.7 | reserved_MB: 28158.0 | inactive_split_MB: 1319.3 +rank7 [2026-10-29 16:17:03,635] INFO : Step 30800/50000 rank7 | Train Loss: 1.0342 | Train Loss Unweighted: 0.4833 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1425 | Classification: 0.0824 | Classification_binary: 0.1755 | HitGrouping: 0.1535 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1102 | Regression_eta: 0.0015 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0626 | Regression_eta: 0.0389 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0795 | LR: 3.90e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0636s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.8308 | max_loss: 0.1717 | max_backward: 2.1170 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 1.1993 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5452.5 | uss_MB: 4327.4 | child_rss_MB: 5889.7 | allocated_MB: 500.1 | reserved_MB: 30158.0 | inactive_split_MB: 868.4 +rank7 [2026-10-29 16:21:38,477] INFO : Step 30900/50000 rank7 | Train Loss: 0.9663 | Train Loss Unweighted: 0.4569 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1376 | Classification: 0.0726 | Classification_binary: 0.1572 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1116 | Regression_eta: 0.0013 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0621 | Regression_eta: 0.0360 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0576 | LR: 3.87e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.6926s | Phase Times: max_pre_step_barrier: 0.0236 | max_batch_to_device: 0.0460 | max_forward: 1.0303 | max_loss: 0.1479 | max_backward: 1.9001 | max_optimizer_step: 0.1728 | max_tensorboard_logging: 1.0216 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5441.4 | uss_MB: 4325.7 | child_rss_MB: 5846.3 | allocated_MB: 495.2 | reserved_MB: 30158.0 | inactive_split_MB: 1414.1 +rank7 [2026-10-29 16:26:02,768] INFO : Step 31000/50000 rank7 | Train Loss: 0.8308 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0939 | Classification: 0.0628 | Classification_binary: 0.1191 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1088 | Regression_eta: 0.0013 | Regression_pt: 0.1096 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0723 | Regression_eta: 0.0406 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0718 | LR: 3.84e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0245s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0072 | max_forward: 0.6940 | max_loss: 0.1397 | max_backward: 1.5452 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.8210 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5428.2 | uss_MB: 4320.0 | child_rss_MB: 5820.2 | allocated_MB: 497.0 | reserved_MB: 30158.0 | inactive_split_MB: 1397.0 +rank7 [2026-10-29 16:26:49,214] INFO : Running validation on rank7 +rank7 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:26:49,214] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:27:39,758] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:27:40,305] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 16:28:07,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:28:09,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:28:11,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:28:12,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:28:23,009] INFO : Time taken to make predictions on device 7 is: 0.29 min +rank7 [2026-10-29 16:28:24,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:28:24,725] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 16:28:40,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:28:41,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:28:42,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:28:43,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:28:53,613] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 16:28:54,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:28:54,677] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 16:29:10,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:29:11,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:29:12,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:29:12,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_31000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:29:22,953] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 16:34:37,556] INFO : Step 31100/50000 rank7 | Train Loss: 0.9765 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1236 | Classification: 0.0781 | Classification_binary: 0.1711 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0507 | Regression_eta: 0.0507 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0882 | LR: 3.82e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.5530s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0951 | max_forward: 1.1933 | max_loss: 0.2819 | max_backward: 2.4156 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 1.1036 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6322.1 | uss_MB: 5090.1 | child_rss_MB: 5840.4 | allocated_MB: 500.1 | reserved_MB: 24414.0 | inactive_split_MB: 1197.9 +rank7 [2026-10-29 16:39:13,624] INFO : Step 31200/50000 rank7 | Train Loss: 0.9956 | Train Loss Unweighted: 0.4667 | Train Loss Components: Auxiliary: 0.2073 | Cardinality: 0.1268 | Classification: 0.0734 | Classification_binary: 0.1778 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1053 | Regression_eta: 0.0017 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0523 | Regression_eta: 0.0350 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0516 | LR: 3.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0726s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.9118 | max_loss: 0.1560 | max_backward: 2.1209 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1559 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6319.5 | uss_MB: 5205.3 | child_rss_MB: 5835.3 | allocated_MB: 495.7 | reserved_MB: 28538.0 | inactive_split_MB: 1120.3 +rank7 [2026-10-29 16:43:57,173] INFO : Step 31300/50000 rank7 | Train Loss: 0.8790 | Train Loss Unweighted: 0.4253 | Train Loss Components: Auxiliary: 0.1786 | Cardinality: 0.1060 | Classification: 0.0761 | Classification_binary: 0.1300 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1082 | Regression_eta: 0.0013 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0567 | Regression_eta: 0.0351 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0448 | LR: 3.76e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8829s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.6947 | max_loss: 0.2409 | max_backward: 1.6530 | max_optimizer_step: 0.1759 | max_tensorboard_logging: 1.3887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5514.1 | uss_MB: 4413.0 | child_rss_MB: 5772.9 | allocated_MB: 495.7 | reserved_MB: 28540.0 | inactive_split_MB: 1092.4 +rank7 [2026-10-29 16:48:30,666] INFO : Step 31400/50000 rank7 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1773 | Cardinality: 0.0849 | Classification: 0.0713 | Classification_binary: 0.1362 | HitGrouping: 0.1550 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1009 | Regression_eta: 0.0015 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1093 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0487 | Regression_eta: 0.0371 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0443 | LR: 3.74e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7847s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.8454 | max_loss: 0.1561 | max_backward: 2.0371 | max_optimizer_step: 0.1732 | max_tensorboard_logging: 1.0195 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5516.9 | uss_MB: 4417.1 | child_rss_MB: 5770.4 | allocated_MB: 497.9 | reserved_MB: 28542.0 | inactive_split_MB: 954.8 +rank7 [2026-10-29 16:53:07,751] INFO : Step 31500/50000 rank7 | Train Loss: 0.9636 | Train Loss Unweighted: 0.4550 | Train Loss Components: Auxiliary: 0.2016 | Cardinality: 0.1194 | Classification: 0.0674 | Classification_binary: 0.1688 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1074 | Regression_eta: 0.0016 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0558 | Regression_eta: 0.0373 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0437 | LR: 3.71e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4930s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8340 | max_loss: 0.1603 | max_backward: 2.1631 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 0.6355 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 5392.3 | uss_MB: 4403.7 | child_rss_MB: 5412.9 | allocated_MB: 498.6 | reserved_MB: 28542.0 | inactive_split_MB: 1182.4 +rank7 [2026-10-29 16:57:38,607] INFO : Step 31600/50000 rank7 | Train Loss: 0.8578 | Train Loss Unweighted: 0.4124 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.0943 | Classification: 0.0661 | Classification_binary: 0.1402 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0012 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0668 | Regression_eta: 0.0282 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0586 | LR: 3.69e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1666s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.8676 | max_loss: 0.1550 | max_backward: 1.8995 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.5800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5355.7 | uss_MB: 4413.3 | child_rss_MB: 5268.1 | allocated_MB: 500.5 | reserved_MB: 28542.0 | inactive_split_MB: 1188.4 +rank7 [2026-10-29 17:02:11,475] INFO : Step 31700/50000 rank7 | Train Loss: 0.8551 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.0825 | Classification: 0.0719 | Classification_binary: 0.1310 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0785 | Regression_eta: 0.0449 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0570 | LR: 3.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1008s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0065 | max_forward: 0.8137 | max_loss: 0.1416 | max_backward: 1.7965 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5293.4 | uss_MB: 4417.4 | child_rss_MB: 5006.3 | allocated_MB: 495.1 | reserved_MB: 28548.0 | inactive_split_MB: 910.4 +rank7 [2026-10-29 17:06:47,524] INFO : Step 31800/50000 rank7 | Train Loss: 0.9445 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1991 | Cardinality: 0.1126 | Classification: 0.0825 | Classification_binary: 0.1632 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1004 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0482 | Regression_eta: 0.0386 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0488 | LR: 3.64e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2351s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.1467 | max_backward: 1.9201 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5286.3 | uss_MB: 4414.9 | child_rss_MB: 4988.0 | allocated_MB: 499.6 | reserved_MB: 28548.0 | inactive_split_MB: 948.4 +rank7 [2026-10-29 17:11:27,372] INFO : Step 31900/50000 rank7 | Train Loss: 0.8948 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.1004 | Classification: 0.0695 | Classification_binary: 0.1514 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0648 | Regression_eta: 0.0345 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0521 | LR: 3.61e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5180s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0823 | max_forward: 0.8585 | max_loss: 0.1528 | max_backward: 1.9830 | max_optimizer_step: 0.1718 | max_tensorboard_logging: 0.6632 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0961 | Memory: rss_MB: 5259.9 | uss_MB: 4415.6 | child_rss_MB: 4865.6 | allocated_MB: 498.9 | reserved_MB: 28552.0 | inactive_split_MB: 963.2 +rank7 [2026-10-29 17:16:11,996] INFO : Step 32000/50000 rank7 | Train Loss: 0.8649 | Train Loss Unweighted: 0.4265 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.0949 | Classification: 0.0664 | Classification_binary: 0.1321 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1119 | Regression_eta: 0.0013 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0650 | Regression_eta: 0.0258 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0642 | LR: 3.58e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4956s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0777 | max_forward: 0.7948 | max_loss: 0.4757 | max_backward: 1.8745 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.8401 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 5240.9 | uss_MB: 4412.4 | child_rss_MB: 4790.7 | allocated_MB: 499.6 | reserved_MB: 28552.0 | inactive_split_MB: 949.8 +rank7 [2026-10-29 17:16:12,936] INFO : Running validation on rank7 +rank7 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 17:16:12,936] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 17:17:03,854] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:17:04,359] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 17:17:32,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 17:17:32,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 17:17:34,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 17:17:35,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 17:17:45,117] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 17:17:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:17:46,318] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 17:18:02,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 17:18:03,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 17:18:04,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 17:18:05,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 17:18:15,582] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 17:18:15,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:18:16,464] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 17:18:31,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 17:18:32,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 17:18:33,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 17:18:34,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_32000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 17:18:44,300] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 17:23:51,376] INFO : Step 32100/50000 rank7 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4359 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1007 | Classification: 0.0791 | Classification_binary: 0.1452 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0590 | Regression_eta: 0.0360 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0727 | LR: 3.56e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9449s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.9334 | max_loss: 0.1454 | max_backward: 2.2729 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.9175 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6428.7 | uss_MB: 5600.2 | child_rss_MB: 4762.7 | allocated_MB: 498.2 | reserved_MB: 23572.0 | inactive_split_MB: 1315.8 +rank7 [2026-10-29 17:28:36,179] INFO : Step 32200/50000 rank7 | Train Loss: 0.9528 | Train Loss Unweighted: 0.4600 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.0993 | Classification: 0.0695 | Classification_binary: 0.1757 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1051 | Regression_eta: 0.0015 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0579 | Regression_eta: 0.0407 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0729 | LR: 3.53e-04 | DataLoad Time: 0.0126s | Model Forward Time: 5.4490s | Phase Times: max_pre_step_barrier: 0.0075 | max_batch_to_device: 0.0166 | max_forward: 1.6389 | max_loss: 0.6243 | max_backward: 2.3737 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6422.8 | uss_MB: 5597.6 | child_rss_MB: 4754.6 | allocated_MB: 497.4 | reserved_MB: 23572.0 | inactive_split_MB: 1351.5 +rank7 [2026-10-29 17:33:21,417] INFO : Step 32300/50000 rank7 | Train Loss: 0.8689 | Train Loss Unweighted: 0.4064 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.1082 | Classification: 0.0618 | Classification_binary: 0.1280 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1068 | Regression_eta: 0.0014 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0609 | Regression_eta: 0.0500 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0434 | LR: 3.51e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9284s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.9017 | max_loss: 0.1622 | max_backward: 2.1045 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 1.2791 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0414 | Memory: rss_MB: 5327.3 | uss_MB: 3764.4 | child_rss_MB: 4755.7 | allocated_MB: 493.8 | reserved_MB: 24702.0 | inactive_split_MB: 1183.9 +rank7 [2026-10-29 17:38:08,351] INFO : Step 32400/50000 rank7 | Train Loss: 0.8527 | Train Loss Unweighted: 0.4031 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1040 | Classification: 0.0677 | Classification_binary: 0.1299 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1013 | Regression_eta: 0.0012 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1125 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0524 | Regression_eta: 0.0221 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0402 | LR: 3.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8164s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7537 | max_loss: 0.1494 | max_backward: 1.6688 | max_optimizer_step: 0.1746 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5333.6 | uss_MB: 4517.8 | child_rss_MB: 4738.7 | allocated_MB: 498.2 | reserved_MB: 24702.0 | inactive_split_MB: 1182.7 +rank7 [2026-10-29 17:42:52,781] INFO : Step 32500/50000 rank7 | Train Loss: 0.9446 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1994 | Cardinality: 0.1079 | Classification: 0.0786 | Classification_binary: 0.1633 | HitGrouping: 0.1495 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1034 | Regression_eta: 0.0013 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0551 | Regression_eta: 0.0454 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0592 | LR: 3.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8566s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.7877 | max_loss: 0.2462 | max_backward: 1.9307 | max_optimizer_step: 0.2560 | max_tensorboard_logging: 1.0800 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 5318.4 | uss_MB: 4505.8 | child_rss_MB: 4711.7 | allocated_MB: 500.3 | reserved_MB: 26774.0 | inactive_split_MB: 1246.9 +rank7 [2026-10-29 17:47:36,596] INFO : Step 32600/50000 rank7 | Train Loss: 0.9260 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1923 | Cardinality: 0.1217 | Classification: 0.0679 | Classification_binary: 0.1580 | HitGrouping: 0.1523 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1025 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0514 | Regression_eta: 0.0301 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0585 | LR: 3.43e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6231s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0066 | max_forward: 0.8137 | max_loss: 0.1538 | max_backward: 1.8655 | max_optimizer_step: 0.1719 | max_tensorboard_logging: 0.8623 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1075 | Memory: rss_MB: 5323.4 | uss_MB: 4518.1 | child_rss_MB: 4716.3 | allocated_MB: 496.0 | reserved_MB: 26774.0 | inactive_split_MB: 1250.1 +rank7 [2026-10-29 17:52:17,188] INFO : Step 32700/50000 rank7 | Train Loss: 0.9240 | Train Loss Unweighted: 0.4306 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1076 | Classification: 0.0629 | Classification_binary: 0.1419 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1111 | Regression_eta: 0.0014 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0489 | Regression_eta: 0.0579 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0663 | LR: 3.41e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5862s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0104 | max_forward: 0.7609 | max_loss: 0.1460 | max_backward: 1.8967 | max_optimizer_step: 0.1753 | max_tensorboard_logging: 0.9783 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5325.1 | uss_MB: 4519.8 | child_rss_MB: 4716.1 | allocated_MB: 497.3 | reserved_MB: 26774.0 | inactive_split_MB: 1245.0 +rank7 [2026-10-29 17:57:01,869] INFO : Step 32800/50000 rank7 | Train Loss: 0.8912 | Train Loss Unweighted: 0.4329 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.0978 | Classification: 0.0863 | Classification_binary: 0.1529 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0947 | Regression_eta: 0.0013 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0436 | Regression_eta: 0.0296 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0343 | LR: 3.38e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7833 | max_loss: 0.1953 | max_backward: 1.9218 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5322.3 | uss_MB: 4519.4 | child_rss_MB: 4692.5 | allocated_MB: 499.4 | reserved_MB: 26774.0 | inactive_split_MB: 957.2 +rank7 [2026-10-29 18:01:38,148] INFO : Step 32900/50000 rank7 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4490 | Train Loss Components: Auxiliary: 0.2061 | Cardinality: 0.1339 | Classification: 0.0697 | Classification_binary: 0.1700 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1029 | Regression_eta: 0.0014 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0609 | Regression_eta: 0.0354 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0671 | LR: 3.36e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.4241s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9108 | max_loss: 0.1579 | max_backward: 2.1063 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5519 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5322.5 | uss_MB: 4520.7 | child_rss_MB: 4706.2 | allocated_MB: 498.2 | reserved_MB: 27252.0 | inactive_split_MB: 1311.8 +rank7 [2026-10-29 18:06:22,611] INFO : Step 33000/50000 rank7 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3978 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.0980 | Classification: 0.0618 | Classification_binary: 0.1213 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0012 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0315 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0509 | Regression_eta: 0.0287 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0349 | LR: 3.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2113s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7275 | max_loss: 0.1602 | max_backward: 1.4819 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0203 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5319.2 | uss_MB: 4517.4 | child_rss_MB: 4715.3 | allocated_MB: 495.1 | reserved_MB: 27252.0 | inactive_split_MB: 1023.1 +rank7 [2026-10-29 18:06:23,517] INFO : Running validation on rank7 +rank7 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:06:23,517] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:07:15,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:07:15,766] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 18:07:43,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:07:44,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:07:46,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:07:47,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:07:57,288] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:07:57,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:07:58,401] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 18:08:14,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:08:15,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:08:16,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:08:17,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:08:27,560] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:08:27,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:08:28,345] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 18:08:43,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:08:44,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:08:45,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:08:46,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_33000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:08:56,185] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 18:13:51,313] INFO : Step 33100/50000 rank7 | Train Loss: 0.8292 | Train Loss Unweighted: 0.3892 | Train Loss Components: Auxiliary: 0.1680 | Cardinality: 0.0939 | Classification: 0.0664 | Classification_binary: 0.1282 | HitGrouping: 0.1504 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0955 | Regression_eta: 0.0014 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0686 | Regression_eta: 0.0431 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0756 | LR: 3.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2611s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7560 | max_loss: 0.2453 | max_backward: 1.9204 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.4836 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1080 | Memory: rss_MB: 6370.4 | uss_MB: 5569.4 | child_rss_MB: 4714.8 | allocated_MB: 495.5 | reserved_MB: 27318.0 | inactive_split_MB: 1128.5 +rank7 [2026-10-29 18:18:33,980] INFO : Step 33200/50000 rank7 | Train Loss: 0.9276 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1970 | Cardinality: 0.1158 | Classification: 0.0748 | Classification_binary: 0.1646 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0971 | Regression_eta: 0.0014 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0473 | Regression_eta: 0.0382 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0621 | LR: 3.28e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4344s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.9248 | max_loss: 0.1506 | max_backward: 2.1567 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.4862 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6371.3 | uss_MB: 5571.3 | child_rss_MB: 4713.6 | allocated_MB: 497.4 | reserved_MB: 27318.0 | inactive_split_MB: 1126.1 +rank7 [2026-10-29 18:23:15,420] INFO : Step 33300/50000 rank7 | Train Loss: 0.8489 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1765 | Cardinality: 0.0998 | Classification: 0.0629 | Classification_binary: 0.1365 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1032 | Regression_eta: 0.0012 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0601 | Regression_eta: 0.0299 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0562 | LR: 3.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6017s | Phase Times: max_pre_step_barrier: 0.0247 | max_batch_to_device: 0.1052 | max_forward: 0.8968 | max_loss: 0.1446 | max_backward: 1.8467 | max_optimizer_step: 0.1835 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 5383.0 | uss_MB: 4585.5 | child_rss_MB: 4710.1 | allocated_MB: 496.7 | reserved_MB: 27318.0 | inactive_split_MB: 1196.7 +rank7 [2026-10-29 18:27:59,290] INFO : Step 33400/50000 rank7 | Train Loss: 0.7914 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0842 | Classification: 0.0704 | Classification_binary: 0.1158 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0963 | Regression_eta: 0.0013 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0336 | Classification_binary: 0.1040 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0486 | Regression_eta: 0.0289 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0560 | LR: 3.23e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.2631s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.7593 | max_loss: 0.1645 | max_backward: 1.8328 | max_optimizer_step: 0.1724 | max_tensorboard_logging: 0.7589 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0193 | Memory: rss_MB: 5391.6 | uss_MB: 4597.3 | child_rss_MB: 4701.5 | allocated_MB: 498.9 | reserved_MB: 27318.0 | inactive_split_MB: 1197.6 +rank7 [2026-10-29 18:32:38,323] INFO : Step 33500/50000 rank7 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4336 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1094 | Classification: 0.0677 | Classification_binary: 0.1630 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0990 | Regression_eta: 0.0015 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0496 | Regression_eta: 0.0400 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0484 | LR: 3.21e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3717s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0077 | max_forward: 0.7864 | max_loss: 0.2197 | max_backward: 1.9118 | max_optimizer_step: 0.2813 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0191 | Memory: rss_MB: 5392.6 | uss_MB: 4599.2 | child_rss_MB: 4701.7 | allocated_MB: 499.9 | reserved_MB: 27318.0 | inactive_split_MB: 939.6 +rank7 [2026-10-29 18:37:27,754] INFO : Step 33600/50000 rank7 | Train Loss: 0.9543 | Train Loss Unweighted: 0.4481 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1202 | Classification: 0.0818 | Classification_binary: 0.1546 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0016 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0484 | Regression_eta: 0.0632 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0663 | LR: 3.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3477s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7980 | max_loss: 0.1984 | max_backward: 1.9347 | max_optimizer_step: 0.2347 | max_tensorboard_logging: 0.6290 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0164 | Memory: rss_MB: 5390.7 | uss_MB: 4597.1 | child_rss_MB: 4698.9 | allocated_MB: 498.3 | reserved_MB: 27318.0 | inactive_split_MB: 1212.1 +rank7 [2026-10-29 18:42:20,045] INFO : Step 33700/50000 rank7 | Train Loss: 0.8364 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.1129 | Classification: 0.0643 | Classification_binary: 0.1208 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0944 | Regression_eta: 0.0012 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1178 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0467 | Regression_eta: 0.0361 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0538 | LR: 3.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3183s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0058 | max_forward: 0.7007 | max_loss: 0.2028 | max_backward: 1.7183 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.9236 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5392.0 | uss_MB: 4598.0 | child_rss_MB: 4710.1 | allocated_MB: 495.2 | reserved_MB: 27326.0 | inactive_split_MB: 1026.0 +rank7 [2026-10-29 18:47:04,987] INFO : Step 33800/50000 rank7 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1837 | Cardinality: 0.0954 | Classification: 0.0791 | Classification_binary: 0.1537 | HitGrouping: 0.1462 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0925 | Regression_eta: 0.0014 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0403 | Regression_eta: 0.0325 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0504 | LR: 3.14e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3962s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1092 | max_forward: 1.1730 | max_loss: 0.1455 | max_backward: 1.8478 | max_optimizer_step: 0.1771 | max_tensorboard_logging: 0.6902 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5392.3 | uss_MB: 4597.5 | child_rss_MB: 4697.8 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 1311.4 +rank7 [2026-10-29 18:51:52,248] INFO : Step 33900/50000 rank7 | Train Loss: 0.8898 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1839 | Cardinality: 0.1142 | Classification: 0.0639 | Classification_binary: 0.1587 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0922 | Regression_eta: 0.0014 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0488 | Regression_eta: 0.0359 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0636 | LR: 3.11e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1928s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7862 | max_loss: 0.1482 | max_backward: 1.7055 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 0.6980 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5393.6 | uss_MB: 4597.6 | child_rss_MB: 4710.1 | allocated_MB: 498.6 | reserved_MB: 27326.0 | inactive_split_MB: 949.2 +rank7 [2026-10-29 18:56:35,808] INFO : Step 34000/50000 rank7 | Train Loss: 0.8429 | Train Loss Unweighted: 0.3992 | Train Loss Components: Auxiliary: 0.1706 | Cardinality: 0.1037 | Classification: 0.0642 | Classification_binary: 0.1243 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1033 | Regression_eta: 0.0013 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0589 | Regression_eta: 0.0276 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0662 | LR: 3.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1641s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0499 | max_forward: 0.7659 | max_loss: 0.1428 | max_backward: 1.8223 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.5832 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 5391.6 | uss_MB: 4596.1 | child_rss_MB: 4708.9 | allocated_MB: 495.9 | reserved_MB: 27326.0 | inactive_split_MB: 1236.1 +rank7 [2026-10-29 18:56:36,807] INFO : Running validation on rank7 +rank7 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:56:36,807] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:57:29,827] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:57:30,355] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 18:57:54,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:57:55,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:57:56,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:57:57,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:58:07,833] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:58:08,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:58:09,212] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 18:58:25,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:58:26,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:58:27,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:58:28,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:58:38,697] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:58:39,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:58:39,751] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 18:58:54,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:58:55,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:58:56,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:58:57,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_34000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:59:07,292] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 19:04:16,498] INFO : Step 34100/50000 rank7 | Train Loss: 0.9167 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.1904 | Cardinality: 0.1190 | Classification: 0.0714 | Classification_binary: 0.1424 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1032 | Regression_eta: 0.0013 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0631 | Regression_eta: 0.0392 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0695 | LR: 3.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7953 | max_loss: 0.2422 | max_backward: 2.1029 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5486.0 | uss_MB: 4688.1 | child_rss_MB: 4696.4 | allocated_MB: 500.1 | reserved_MB: 23134.0 | inactive_split_MB: 1095.9 +rank7 [2026-10-29 19:09:14,637] INFO : Step 34200/50000 rank7 | Train Loss: 0.9128 | Train Loss Unweighted: 0.4402 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1076 | Classification: 0.0741 | Classification_binary: 0.1584 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1020 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0473 | Regression_eta: 0.0369 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0512 | LR: 3.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6992s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9174 | max_loss: 0.1566 | max_backward: 2.1253 | max_optimizer_step: 0.1788 | max_tensorboard_logging: 0.7432 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0604 | Memory: rss_MB: 5494.0 | uss_MB: 4698.3 | child_rss_MB: 4711.4 | allocated_MB: 497.7 | reserved_MB: 27130.0 | inactive_split_MB: 1299.8 +rank7 [2026-10-29 19:14:01,731] INFO : Step 34300/50000 rank7 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1022 | Classification: 0.0604 | Classification_binary: 0.1442 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0472 | Regression_eta: 0.0414 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0715 | LR: 3.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3187s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0208 | max_forward: 0.8028 | max_loss: 0.1849 | max_backward: 1.9797 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.5895 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5480.7 | uss_MB: 4686.1 | child_rss_MB: 4716.9 | allocated_MB: 494.7 | reserved_MB: 27136.0 | inactive_split_MB: 1035.4 +rank7 [2026-10-29 19:18:48,203] INFO : Step 34400/50000 rank7 | Train Loss: 0.8270 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1714 | Cardinality: 0.0900 | Classification: 0.0726 | Classification_binary: 0.1195 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0342 | Regression_eta: 0.0428 | Regression_pt: 0.0351 | Regression_sin_phi: 0.0617 | LR: 2.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9240s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7048 | max_loss: 0.1341 | max_backward: 1.5625 | max_optimizer_step: 0.1791 | max_tensorboard_logging: 0.6646 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5492.0 | uss_MB: 4696.7 | child_rss_MB: 4713.1 | allocated_MB: 498.5 | reserved_MB: 27136.0 | inactive_split_MB: 1073.5 +rank7 [2026-10-29 19:23:49,388] INFO : Step 34500/50000 rank7 | Train Loss: 0.9162 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1963 | Cardinality: 0.1155 | Classification: 0.0749 | Classification_binary: 0.1529 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0970 | Regression_eta: 0.0013 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0485 | Regression_eta: 0.0356 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0437 | LR: 2.97e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.2315s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8204 | max_loss: 0.1592 | max_backward: 1.9717 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.4730 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5484.9 | uss_MB: 4699.6 | child_rss_MB: 4714.3 | allocated_MB: 498.3 | reserved_MB: 31260.0 | inactive_split_MB: 1303.3 +rank7 [2026-10-29 19:28:34,774] INFO : Step 34600/50000 rank7 | Train Loss: 0.9076 | Train Loss Unweighted: 0.4214 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1204 | Classification: 0.0699 | Classification_binary: 0.1572 | HitGrouping: 0.1438 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0951 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1403 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0540 | Regression_eta: 0.0444 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0465 | LR: 2.95e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4737s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.9474 | max_loss: 0.1550 | max_backward: 2.2571 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4697 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5478.3 | uss_MB: 4698.2 | child_rss_MB: 4711.3 | allocated_MB: 499.6 | reserved_MB: 31260.0 | inactive_split_MB: 1162.4 +rank7 [2026-10-29 19:33:35,742] INFO : Step 34700/50000 rank7 | Train Loss: 0.7952 | Train Loss Unweighted: 0.3766 | Train Loss Components: Auxiliary: 0.1627 | Cardinality: 0.0942 | Classification: 0.0688 | Classification_binary: 0.1209 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0920 | Regression_eta: 0.0012 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0447 | Regression_eta: 0.0368 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0598 | LR: 2.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1490s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.8096 | max_loss: 0.1455 | max_backward: 1.6758 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0219 | Memory: rss_MB: 5471.7 | uss_MB: 4696.2 | child_rss_MB: 4717.0 | allocated_MB: 498.5 | reserved_MB: 31260.0 | inactive_split_MB: 1415.1 +rank7 [2026-10-29 19:38:26,589] INFO : Step 34800/50000 rank7 | Train Loss: 0.8539 | Train Loss Unweighted: 0.3971 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1116 | Classification: 0.0753 | Classification_binary: 0.1336 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0923 | Regression_eta: 0.0013 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1601 | Regression_cos_phi: 0.1113 | Regression_energy: 0.0553 | Regression_eta: 0.0529 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0736 | LR: 2.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0553s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0076 | max_forward: 0.7791 | max_loss: 0.1579 | max_backward: 1.7752 | max_optimizer_step: 0.1823 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0894 | Memory: rss_MB: 5461.4 | uss_MB: 4696.2 | child_rss_MB: 4712.7 | allocated_MB: 501.8 | reserved_MB: 31260.0 | inactive_split_MB: 1426.8 +rank7 [2026-10-29 19:43:15,574] INFO : Step 34900/50000 rank7 | Train Loss: 0.9024 | Train Loss Unweighted: 0.4187 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1178 | Classification: 0.0659 | Classification_binary: 0.1638 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0927 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0653 | Regression_eta: 0.0469 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0638 | LR: 2.88e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2774s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.2271 | max_backward: 1.8377 | max_optimizer_step: 0.2662 | max_tensorboard_logging: 0.5737 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 5454.3 | uss_MB: 4696.7 | child_rss_MB: 4712.5 | allocated_MB: 499.0 | reserved_MB: 31262.0 | inactive_split_MB: 1107.8 +rank7 [2026-10-29 19:48:20,349] INFO : Step 35000/50000 rank7 | Train Loss: 0.8846 | Train Loss Unweighted: 0.4004 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1188 | Classification: 0.0646 | Classification_binary: 0.1355 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0986 | Regression_eta: 0.0013 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0550 | Regression_eta: 0.0447 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0560 | LR: 2.85e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3860s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.9193 | max_loss: 0.1475 | max_backward: 2.1298 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.4651 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5454.7 | uss_MB: 4698.6 | child_rss_MB: 4709.6 | allocated_MB: 498.6 | reserved_MB: 31262.0 | inactive_split_MB: 1188.8 +rank7 [2026-10-29 19:48:21,335] INFO : Running validation on rank7 +rank7 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 19:48:21,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 19:49:15,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:49:15,988] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 19:49:42,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 19:49:43,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 19:49:44,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 19:49:45,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 19:49:55,701] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:49:56,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:49:56,734] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 19:50:12,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 19:50:13,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 19:50:14,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 19:50:15,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 19:50:25,822] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:50:26,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:50:26,882] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 19:50:42,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 19:50:43,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 19:50:44,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 19:50:44,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_35000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 19:50:54,874] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 19:56:07,676] INFO : Step 35100/50000 rank7 | Train Loss: 0.8388 | Train Loss Unweighted: 0.3848 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.1014 | Classification: 0.0691 | Classification_binary: 0.1218 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0308 | Regression_energy: 0.0549 | Regression_eta: 0.0399 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0506 | LR: 2.83e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4696s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7535 | max_loss: 0.4791 | max_backward: 1.9233 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.7608 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5545.2 | uss_MB: 4781.1 | child_rss_MB: 4671.5 | allocated_MB: 497.4 | reserved_MB: 25612.0 | inactive_split_MB: 1221.5 +rank7 [2026-10-29 20:01:01,618] INFO : Step 35200/50000 rank7 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0695 | Classification_binary: 0.1525 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0586 | Regression_eta: 0.0277 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0402 | LR: 2.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6708s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8067 | max_loss: 0.2333 | max_backward: 2.1928 | max_optimizer_step: 0.1813 | max_tensorboard_logging: 0.6201 | max_tensorboard_flush: 0.0010 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5554.1 | uss_MB: 4793.3 | child_rss_MB: 4673.4 | allocated_MB: 498.6 | reserved_MB: 25614.0 | inactive_split_MB: 1034.8 +rank7 [2026-10-29 20:05:52,155] INFO : Step 35300/50000 rank7 | Train Loss: 0.9330 | Train Loss Unweighted: 0.4324 | Train Loss Components: Auxiliary: 0.1978 | Cardinality: 0.1414 | Classification: 0.0643 | Classification_binary: 0.1469 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0615 | Regression_eta: 0.0405 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0724 | LR: 2.79e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1448s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0111 | max_forward: 0.7687 | max_loss: 0.1518 | max_backward: 1.8895 | max_optimizer_step: 0.1815 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5548.3 | uss_MB: 4790.6 | child_rss_MB: 4666.6 | allocated_MB: 495.0 | reserved_MB: 25614.0 | inactive_split_MB: 1097.1 +rank7 [2026-10-29 20:10:54,439] INFO : Step 35400/50000 rank7 | Train Loss: 0.8129 | Train Loss Unweighted: 0.3774 | Train Loss Components: Auxiliary: 0.1662 | Cardinality: 0.1007 | Classification: 0.0630 | Classification_binary: 0.1164 | HitGrouping: 0.1520 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0011 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0217 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0472 | Regression_eta: 0.0230 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0235 | LR: 2.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4638s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1072 | max_forward: 0.7955 | max_loss: 0.1429 | max_backward: 2.0698 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.6336 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5549.2 | uss_MB: 4790.9 | child_rss_MB: 4676.8 | allocated_MB: 494.2 | reserved_MB: 27650.0 | inactive_split_MB: 1233.5 +rank7 [2026-10-29 20:15:42,289] INFO : Step 35500/50000 rank7 | Train Loss: 0.8517 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1828 | Cardinality: 0.1122 | Classification: 0.0624 | Classification_binary: 0.1347 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1212 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0655 | Regression_eta: 0.0286 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0374 | LR: 2.74e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3583s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7722 | max_loss: 0.2448 | max_backward: 1.9131 | max_optimizer_step: 0.2697 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5550.3 | uss_MB: 4791.4 | child_rss_MB: 4678.5 | allocated_MB: 497.0 | reserved_MB: 29810.0 | inactive_split_MB: 1237.0 +rank7 [2026-10-29 20:20:38,046] INFO : Step 35600/50000 rank7 | Train Loss: 0.8994 | Train Loss Unweighted: 0.4276 | Train Loss Components: Auxiliary: 0.1918 | Cardinality: 0.1072 | Classification: 0.0726 | Classification_binary: 0.1557 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0982 | Regression_eta: 0.0014 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0513 | Regression_eta: 0.0326 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0553 | LR: 2.72e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1915s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.8547 | max_loss: 0.1513 | max_backward: 1.8588 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5549 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5547.9 | uss_MB: 4787.8 | child_rss_MB: 4681.7 | allocated_MB: 499.2 | reserved_MB: 29810.0 | inactive_split_MB: 1098.7 +rank7 [2026-10-29 20:25:33,137] INFO : Step 35700/50000 rank7 | Train Loss: 0.8262 | Train Loss Unweighted: 0.3798 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1102 | Classification: 0.0698 | Classification_binary: 0.1202 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0931 | Regression_eta: 0.0012 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0315 | Regression_energy: 0.0358 | Regression_eta: 0.0197 | Regression_pt: 0.0370 | Regression_sin_phi: 0.0366 | LR: 2.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1730s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0704 | max_forward: 0.7632 | max_loss: 0.1407 | max_backward: 1.8337 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.5724 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5549.9 | uss_MB: 4789.7 | child_rss_MB: 4674.8 | allocated_MB: 496.2 | reserved_MB: 29810.0 | inactive_split_MB: 1045.8 +rank7 [2026-10-29 20:30:22,090] INFO : Step 35800/50000 rank7 | Train Loss: 0.8298 | Train Loss Unweighted: 0.4050 | Train Loss Components: Auxiliary: 0.1749 | Cardinality: 0.0860 | Classification: 0.0824 | Classification_binary: 0.1337 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0013 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0506 | Regression_eta: 0.0512 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0601 | LR: 2.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2791s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7595 | max_loss: 0.2041 | max_backward: 1.8742 | max_optimizer_step: 0.2711 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5548.8 | uss_MB: 4790.1 | child_rss_MB: 4672.4 | allocated_MB: 495.9 | reserved_MB: 29810.0 | inactive_split_MB: 1238.7 +rank7 [2026-10-29 20:35:20,188] INFO : Step 35900/50000 rank7 | Train Loss: 0.9271 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1097 | Classification: 0.0707 | Classification_binary: 0.1609 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1080 | Regression_eta: 0.0013 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0495 | Regression_eta: 0.0512 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0624 | LR: 2.65e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6679s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9205 | max_loss: 0.4649 | max_backward: 2.1685 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1033 | Memory: rss_MB: 5549.1 | uss_MB: 4793.1 | child_rss_MB: 4672.4 | allocated_MB: 499.8 | reserved_MB: 29810.0 | inactive_split_MB: 968.1 +rank7 [2026-10-29 20:40:16,075] INFO : Step 36000/50000 rank7 | Train Loss: 0.8271 | Train Loss Unweighted: 0.3970 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0968 | Classification: 0.0672 | Classification_binary: 0.1361 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0950 | Regression_eta: 0.0011 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0465 | Regression_eta: 0.0310 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0594 | LR: 2.63e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2575s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 1.1089 | max_loss: 0.1476 | max_backward: 1.9359 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5547.0 | uss_MB: 4789.8 | child_rss_MB: 4674.7 | allocated_MB: 496.8 | reserved_MB: 29810.0 | inactive_split_MB: 1214.3 +rank7 [2026-10-29 20:40:17,059] INFO : Running validation on rank7 +rank7 [2026-10-29 20:40:17,059] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 20:40:17,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 20:41:08,401] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:41:08,960] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 20:41:36,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 20:41:37,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 20:41:38,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 20:41:39,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 20:41:49,128] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 20:41:49,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:41:50,341] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 20:42:06,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 20:42:07,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 20:42:07,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 20:42:08,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 20:42:18,957] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 20:42:20,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:42:20,694] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 20:42:36,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 20:42:37,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 20:42:37,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 20:42:38,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_36000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 20:42:48,820] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 20:47:46,988] INFO : Step 36100/50000 rank7 | Train Loss: 0.7990 | Train Loss Unweighted: 0.3893 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0876 | Classification: 0.0613 | Classification_binary: 0.1177 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1105 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0656 | Regression_eta: 0.0343 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0460 | LR: 2.61e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4493s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7662 | max_loss: 0.1439 | max_backward: 1.8763 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.6903 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 5622.6 | uss_MB: 4861.4 | child_rss_MB: 4708.8 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1076.6 +rank7 [2026-10-29 20:52:22,841] INFO : Step 36200/50000 rank7 | Train Loss: 0.8859 | Train Loss Unweighted: 0.4068 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1030 | Classification: 0.0698 | Classification_binary: 0.1491 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0918 | Regression_eta: 0.0014 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0531 | Regression_eta: 0.0271 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0504 | LR: 2.59e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3968s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.8584 | max_loss: 0.1554 | max_backward: 1.9383 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5818 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5628.4 | uss_MB: 4869.7 | child_rss_MB: 4684.4 | allocated_MB: 498.3 | reserved_MB: 28784.0 | inactive_split_MB: 1250.4 +rank7 [2026-10-29 20:56:59,584] INFO : Step 36300/50000 rank7 | Train Loss: 0.8677 | Train Loss Unweighted: 0.4284 | Train Loss Components: Auxiliary: 0.1861 | Cardinality: 0.1007 | Classification: 0.0701 | Classification_binary: 0.1509 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1022 | Regression_eta: 0.0011 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0519 | Regression_eta: 0.0308 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0535 | LR: 2.57e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2275s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0933 | max_forward: 0.7931 | max_loss: 0.1422 | max_backward: 1.8596 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5196 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5628.6 | uss_MB: 4868.5 | child_rss_MB: 4706.4 | allocated_MB: 494.7 | reserved_MB: 28784.0 | inactive_split_MB: 1199.9 +rank7 [2026-10-29 21:01:38,351] INFO : Step 36400/50000 rank7 | Train Loss: 0.7939 | Train Loss Unweighted: 0.3883 | Train Loss Components: Auxiliary: 0.1646 | Cardinality: 0.0848 | Classification: 0.0666 | Classification_binary: 0.1202 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0991 | Regression_eta: 0.0011 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0539 | Regression_eta: 0.0421 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0888 | LR: 2.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8757s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7132 | max_loss: 0.1723 | max_backward: 1.6734 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.4801 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5622.9 | uss_MB: 4868.7 | child_rss_MB: 4690.7 | allocated_MB: 493.6 | reserved_MB: 28784.0 | inactive_split_MB: 1033.4 +rank7 [2026-10-29 21:06:14,245] INFO : Step 36500/50000 rank7 | Train Loss: 0.8196 | Train Loss Unweighted: 0.3868 | Train Loss Components: Auxiliary: 0.1717 | Cardinality: 0.0978 | Classification: 0.0656 | Classification_binary: 0.1377 | HitGrouping: 0.1395 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1291 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0372 | Regression_eta: 0.0301 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0346 | LR: 2.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4212s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8844 | max_loss: 0.4683 | max_backward: 2.1003 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0243 | Memory: rss_MB: 5623.5 | uss_MB: 4874.0 | child_rss_MB: 4667.6 | allocated_MB: 500.3 | reserved_MB: 28784.0 | inactive_split_MB: 1244.4 +rank7 [2026-10-29 21:10:49,731] INFO : Step 36600/50000 rank7 | Train Loss: 0.8891 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1124 | Classification: 0.0709 | Classification_binary: 0.1515 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0973 | Regression_eta: 0.0013 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0470 | Regression_eta: 0.0366 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0519 | LR: 2.50e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2770s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0072 | max_forward: 0.8212 | max_loss: 0.1561 | max_backward: 1.9693 | max_optimizer_step: 0.1842 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5620.5 | uss_MB: 4872.3 | child_rss_MB: 4685.7 | allocated_MB: 499.3 | reserved_MB: 28784.0 | inactive_split_MB: 1093.9 +rank7 [2026-10-29 21:15:35,108] INFO : Step 36700/50000 rank7 | Train Loss: 0.7898 | Train Loss Unweighted: 0.3738 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0847 | Classification: 0.0713 | Classification_binary: 0.1160 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0918 | Regression_eta: 0.0013 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0487 | Regression_eta: 0.0319 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0653 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2690s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7836 | max_loss: 0.2578 | max_backward: 1.9588 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.4899 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0775 | Memory: rss_MB: 5612.4 | uss_MB: 4870.5 | child_rss_MB: 4677.1 | allocated_MB: 495.5 | reserved_MB: 29466.0 | inactive_split_MB: 1095.1 +rank7 [2026-10-29 21:20:28,003] INFO : Step 36800/50000 rank7 | Train Loss: 0.8897 | Train Loss Unweighted: 0.4246 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.1014 | Classification: 0.0659 | Classification_binary: 0.1387 | HitGrouping: 0.1566 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1081 | Regression_eta: 0.0014 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1119 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0401 | Regression_eta: 0.0612 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0493 | LR: 2.46e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3982s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8922 | max_loss: 0.1487 | max_backward: 2.1289 | max_optimizer_step: 0.1859 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 5587.2 | uss_MB: 4872.0 | child_rss_MB: 4605.6 | allocated_MB: 499.6 | reserved_MB: 29466.0 | inactive_split_MB: 1065.9 +rank7 [2026-10-29 21:25:23,681] INFO : Step 36900/50000 rank7 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4326 | Train Loss Components: Auxiliary: 0.1988 | Cardinality: 0.1219 | Classification: 0.0695 | Classification_binary: 0.1622 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0989 | Regression_eta: 0.0012 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0592 | Regression_eta: 0.0388 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0567 | LR: 2.44e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.9344s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0078 | max_forward: 0.9685 | max_loss: 0.1890 | max_backward: 2.3660 | max_optimizer_step: 0.2892 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 5578.2 | uss_MB: 4872.4 | child_rss_MB: 4562.6 | allocated_MB: 498.9 | reserved_MB: 29466.0 | inactive_split_MB: 1251.7 +rank7 [2026-10-29 21:30:16,337] INFO : Step 37000/50000 rank7 | Train Loss: 0.8679 | Train Loss Unweighted: 0.4039 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1182 | Classification: 0.0636 | Classification_binary: 0.1465 | HitGrouping: 0.1486 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0954 | Regression_eta: 0.0012 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1536 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0579 | Regression_eta: 0.0412 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0711 | LR: 2.42e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.9285s | Phase Times: max_pre_step_barrier: 0.0079 | max_batch_to_device: 0.0177 | max_forward: 1.5483 | max_loss: 0.5823 | max_backward: 3.4538 | max_optimizer_step: 0.1825 | max_tensorboard_logging: 0.5317 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5572.1 | uss_MB: 4869.3 | child_rss_MB: 4551.5 | allocated_MB: 496.6 | reserved_MB: 29466.0 | inactive_split_MB: 1000.9 +rank7 [2026-10-29 21:30:17,323] INFO : Running validation on rank7 +rank7 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:30:17,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:31:08,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:31:08,621] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 21:31:35,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:31:36,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:31:37,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:31:38,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:31:48,629] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:31:49,070] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:31:49,595] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 21:32:06,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:32:07,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:32:08,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:32:09,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:32:19,653] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 21:32:20,307] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:32:20,812] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 21:32:35,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:32:36,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:32:37,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:32:38,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_37000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:32:48,379] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 21:37:56,197] INFO : Step 37100/50000 rank7 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3645 | Train Loss Components: Auxiliary: 0.1622 | Cardinality: 0.0994 | Classification: 0.0635 | Classification_binary: 0.1140 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0228 | Classification_binary: 0.1046 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0454 | Regression_eta: 0.0294 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0319 | LR: 2.40e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.6906s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0150 | max_forward: 0.7465 | max_loss: 0.5120 | max_backward: 1.7935 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 1.0282 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0440 | Memory: rss_MB: 7037.4 | uss_MB: 6329.5 | child_rss_MB: 4536.6 | allocated_MB: 495.6 | reserved_MB: 27950.0 | inactive_split_MB: 1173.1 +rank7 [2026-10-29 21:43:23,660] INFO : Step 37200/50000 rank7 | Train Loss: 0.8282 | Train Loss Unweighted: 0.3828 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.1181 | Classification: 0.0695 | Classification_binary: 0.1341 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0538 | Regression_eta: 0.0231 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0318 | LR: 2.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6951s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7623 | max_loss: 0.1702 | max_backward: 1.8033 | max_optimizer_step: 0.2891 | max_tensorboard_logging: 0.9551 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5698.7 | uss_MB: 4990.9 | child_rss_MB: 4508.6 | allocated_MB: 499.0 | reserved_MB: 28056.0 | inactive_split_MB: 1098.8 +rank7 [2026-10-29 21:48:12,155] INFO : Step 37300/50000 rank7 | Train Loss: 0.8972 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1251 | Classification: 0.0658 | Classification_binary: 0.1512 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0012 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0260 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0436 | Regression_eta: 0.0318 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0356 | LR: 2.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5584s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7552 | max_loss: 0.2542 | max_backward: 1.8372 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 0.9146 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5694.4 | uss_MB: 4988.6 | child_rss_MB: 4534.3 | allocated_MB: 498.0 | reserved_MB: 30064.0 | inactive_split_MB: 962.0 +rank7 [2026-10-29 21:52:56,911] INFO : Step 37400/50000 rank7 | Train Loss: 0.7608 | Train Loss Unweighted: 0.3573 | Train Loss Components: Auxiliary: 0.1591 | Cardinality: 0.0874 | Classification: 0.0575 | Classification_binary: 0.1126 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0919 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0527 | Regression_eta: 0.0248 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0607 | LR: 2.34e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6117s | Phase Times: max_pre_step_barrier: 0.0183 | max_batch_to_device: 0.0181 | max_forward: 0.8674 | max_loss: 0.2755 | max_backward: 1.8508 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8168 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 5690.6 | uss_MB: 4986.7 | child_rss_MB: 4536.9 | allocated_MB: 498.7 | reserved_MB: 30064.0 | inactive_split_MB: 1012.4 +rank7 [2026-10-29 21:57:43,878] INFO : Step 37500/50000 rank7 | Train Loss: 0.8512 | Train Loss Unweighted: 0.4095 | Train Loss Components: Auxiliary: 0.1772 | Cardinality: 0.0926 | Classification: 0.0807 | Classification_binary: 0.1389 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0933 | Regression_eta: 0.0015 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0487 | Regression_eta: 0.0452 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0601 | LR: 2.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7109s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0934 | max_forward: 0.8114 | max_loss: 0.1429 | max_backward: 1.9741 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.7417 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 5691.9 | uss_MB: 4989.9 | child_rss_MB: 4538.0 | allocated_MB: 499.3 | reserved_MB: 30064.0 | inactive_split_MB: 1098.8 +rank7 [2026-10-29 22:02:29,841] INFO : Step 37600/50000 rank7 | Train Loss: 0.8637 | Train Loss Unweighted: 0.4037 | Train Loss Components: Auxiliary: 0.1829 | Cardinality: 0.1134 | Classification: 0.0683 | Classification_binary: 0.1521 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0899 | Regression_eta: 0.0012 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0464 | Regression_eta: 0.0305 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0616 | LR: 2.30e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.2495s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.1036 | max_loss: 0.1567 | max_backward: 1.8788 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.4633 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5694.6 | uss_MB: 4990.4 | child_rss_MB: 4527.9 | allocated_MB: 500.5 | reserved_MB: 30064.0 | inactive_split_MB: 1092.9 +rank7 [2026-10-29 22:07:05,609] INFO : Step 37700/50000 rank7 | Train Loss: 0.8257 | Train Loss Unweighted: 0.3746 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1051 | Classification: 0.0630 | Classification_binary: 0.1248 | HitGrouping: 0.1515 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0015 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0555 | Regression_eta: 0.0411 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0457 | LR: 2.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1558s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0200 | max_forward: 0.7600 | max_loss: 0.1569 | max_backward: 1.8796 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.4911 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5694.3 | uss_MB: 4989.6 | child_rss_MB: 4530.3 | allocated_MB: 495.8 | reserved_MB: 30064.0 | inactive_split_MB: 968.5 +rank7 [2026-10-29 22:11:38,774] INFO : Step 37800/50000 rank7 | Train Loss: 0.8605 | Train Loss Unweighted: 0.3977 | Train Loss Components: Auxiliary: 0.1768 | Cardinality: 0.1096 | Classification: 0.0757 | Classification_binary: 0.1261 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0965 | Regression_eta: 0.0014 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1030 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0421 | Regression_eta: 0.0611 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0594 | LR: 2.26e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2566s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7673 | max_loss: 0.1394 | max_backward: 1.9676 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5693.6 | uss_MB: 4989.4 | child_rss_MB: 4528.0 | allocated_MB: 498.9 | reserved_MB: 30064.0 | inactive_split_MB: 1104.4 +rank7 [2026-10-29 22:16:15,813] INFO : Step 37900/50000 rank7 | Train Loss: 0.8856 | Train Loss Unweighted: 0.4104 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1233 | Classification: 0.0694 | Classification_binary: 0.1517 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0347 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0457 | Regression_eta: 0.0399 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0365 | LR: 2.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1258s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7660 | max_loss: 0.2528 | max_backward: 1.8326 | max_optimizer_step: 0.1885 | max_tensorboard_logging: 0.4670 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5688.2 | uss_MB: 4987.7 | child_rss_MB: 4509.9 | allocated_MB: 496.5 | reserved_MB: 30064.0 | inactive_split_MB: 1174.7 +rank7 [2026-10-29 22:20:56,948] INFO : Step 38000/50000 rank7 | Train Loss: 0.8316 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1686 | Cardinality: 0.0925 | Classification: 0.0635 | Classification_binary: 0.1387 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0915 | Regression_eta: 0.0016 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0379 | Regression_eta: 0.0390 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0590 | LR: 2.22e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2880s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7710 | max_loss: 0.2557 | max_backward: 1.8665 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.5907 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5671.0 | uss_MB: 4978.6 | child_rss_MB: 4529.7 | allocated_MB: 499.1 | reserved_MB: 30064.0 | inactive_split_MB: 1317.2 +rank7 [2026-10-29 22:20:57,876] INFO : Running validation on rank7 +rank7 [2026-10-29 22:20:57,876] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 22:20:57,877] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 22:21:48,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:21:48,921] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 22:22:15,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 22:22:16,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 22:22:17,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 22:22:18,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 22:22:28,184] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 22:22:28,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:22:29,428] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 22:22:45,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 22:22:46,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 22:22:47,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 22:22:48,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 22:22:58,228] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 22:22:58,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:22:59,394] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 22:23:14,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 22:23:15,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 22:23:16,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 22:23:16,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_38000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 22:23:26,944] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 22:28:21,527] INFO : Step 38100/50000 rank7 | Train Loss: 0.8179 | Train Loss Unweighted: 0.3948 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.1052 | Classification: 0.0645 | Classification_binary: 0.1206 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1035 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0573 | Regression_eta: 0.0378 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0372 | LR: 2.20e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2307s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7587 | max_loss: 0.1430 | max_backward: 1.9030 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.4877 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 7017.8 | uss_MB: 6320.6 | child_rss_MB: 4534.3 | allocated_MB: 498.7 | reserved_MB: 28754.0 | inactive_split_MB: 1065.3 +rank7 [2026-10-29 22:33:07,240] INFO : Step 38200/50000 rank7 | Train Loss: 0.8444 | Train Loss Unweighted: 0.3904 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.0978 | Classification: 0.0720 | Classification_binary: 0.1430 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0013 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0447 | Regression_eta: 0.0245 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0560 | LR: 2.18e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7606s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0074 | max_forward: 0.9456 | max_loss: 0.5129 | max_backward: 2.2258 | max_optimizer_step: 0.1909 | max_tensorboard_logging: 0.5988 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1189 | Memory: rss_MB: 5735.2 | uss_MB: 5045.1 | child_rss_MB: 4539.9 | allocated_MB: 499.1 | reserved_MB: 30172.0 | inactive_split_MB: 954.9 +rank7 [2026-10-29 22:37:42,242] INFO : Step 38300/50000 rank7 | Train Loss: 0.9061 | Train Loss Unweighted: 0.4149 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1278 | Classification: 0.0688 | Classification_binary: 0.1511 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0957 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0569 | Regression_eta: 0.0293 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0691 | LR: 2.16e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 1.1148 | max_loss: 0.1598 | max_backward: 2.0968 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.6603 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5732.0 | uss_MB: 5044.1 | child_rss_MB: 4543.0 | allocated_MB: 500.5 | reserved_MB: 30172.0 | inactive_split_MB: 1118.4 +rank7 [2026-10-29 22:42:17,609] INFO : Step 38400/50000 rank7 | Train Loss: 0.8168 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1693 | Cardinality: 0.0954 | Classification: 0.0702 | Classification_binary: 0.1173 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1000 | Regression_eta: 0.0013 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1080 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0412 | Regression_eta: 0.0275 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0512 | LR: 2.14e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4886s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0056 | max_forward: 0.7225 | max_loss: 0.2512 | max_backward: 1.7915 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.9241 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5730.5 | uss_MB: 5040.7 | child_rss_MB: 4543.1 | allocated_MB: 494.6 | reserved_MB: 32188.0 | inactive_split_MB: 1006.9 +rank7 [2026-10-29 22:46:52,193] INFO : Step 38500/50000 rank7 | Train Loss: 0.7986 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.1022 | Classification: 0.0641 | Classification_binary: 0.1257 | HitGrouping: 0.1419 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0011 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0520 | Regression_eta: 0.0276 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0372 | LR: 2.12e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5200s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2206 | max_backward: 1.8273 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9231 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5732.7 | uss_MB: 5044.3 | child_rss_MB: 4542.2 | allocated_MB: 497.9 | reserved_MB: 32188.0 | inactive_split_MB: 1005.6 +rank7 [2026-10-29 22:51:33,792] INFO : Step 38600/50000 rank7 | Train Loss: 0.8963 | Train Loss Unweighted: 0.4035 | Train Loss Components: Auxiliary: 0.1902 | Cardinality: 0.1166 | Classification: 0.0627 | Classification_binary: 0.1614 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0877 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0546 | Regression_eta: 0.0395 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0741 | LR: 2.11e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.7269s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8142 | max_loss: 0.2563 | max_backward: 1.9988 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.8297 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5731.8 | uss_MB: 5043.8 | child_rss_MB: 4544.6 | allocated_MB: 497.1 | reserved_MB: 32874.0 | inactive_split_MB: 970.9 +rank7 [2026-10-29 22:56:38,192] INFO : Step 38700/50000 rank7 | Train Loss: 0.8075 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.1167 | Classification: 0.0554 | Classification_binary: 0.1277 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0424 | Regression_eta: 0.0256 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0399 | LR: 2.09e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1555s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7853 | max_loss: 0.1912 | max_backward: 1.7852 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5730.4 | uss_MB: 5042.4 | child_rss_MB: 4546.1 | allocated_MB: 496.2 | reserved_MB: 32874.0 | inactive_split_MB: 1011.1 +rank7 [2026-10-29 23:02:19,420] INFO : Step 38800/50000 rank7 | Train Loss: 0.7998 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1661 | Cardinality: 0.0923 | Classification: 0.0629 | Classification_binary: 0.1125 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1041 | Regression_eta: 0.0010 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1096 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0651 | Regression_eta: 0.0499 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0585 | LR: 2.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3989s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.8635 | max_loss: 0.1389 | max_backward: 1.9722 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5887 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5729.0 | uss_MB: 5042.8 | child_rss_MB: 4543.0 | allocated_MB: 494.7 | reserved_MB: 32874.0 | inactive_split_MB: 928.4 +rank7 [2026-10-29 23:07:29,432] INFO : Step 38900/50000 rank7 | Train Loss: 0.8451 | Train Loss Unweighted: 0.3924 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.1038 | Classification: 0.0623 | Classification_binary: 0.1467 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0904 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0558 | Regression_eta: 0.0330 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0450 | LR: 2.05e-04 | DataLoad Time: 0.0134s | Model Forward Time: 4.8886s | Phase Times: max_pre_step_barrier: 0.0209 | max_batch_to_device: 0.0194 | max_forward: 1.5760 | max_loss: 0.4960 | max_backward: 2.1289 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.4755 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 5729.7 | uss_MB: 5042.9 | child_rss_MB: 4525.4 | allocated_MB: 498.7 | reserved_MB: 32874.0 | inactive_split_MB: 1070.2 +rank7 [2026-10-29 23:12:37,975] INFO : Step 39000/50000 rank7 | Train Loss: 0.8109 | Train Loss Unweighted: 0.3771 | Train Loss Components: Auxiliary: 0.1738 | Cardinality: 0.1141 | Classification: 0.0620 | Classification_binary: 0.1301 | HitGrouping: 0.1383 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0011 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0494 | Regression_eta: 0.0208 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0387 | LR: 2.03e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4694s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1938 | max_backward: 1.8159 | max_optimizer_step: 0.2934 | max_tensorboard_logging: 0.7254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 5718.8 | uss_MB: 5033.0 | child_rss_MB: 4537.6 | allocated_MB: 497.4 | reserved_MB: 32874.0 | inactive_split_MB: 1012.4 +rank7 [2026-10-29 23:12:38,866] INFO : Running validation on rank7 +rank7 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 23:12:38,866] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 23:13:33,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:13:34,231] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-29 23:14:06,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 23:14:07,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 23:14:08,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 23:14:09,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 23:14:19,490] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 23:14:20,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:14:20,782] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-29 23:14:36,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 23:14:37,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 23:14:38,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 23:14:39,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 23:14:49,400] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 23:14:50,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:14:50,898] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-29 23:15:06,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 23:15:07,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 23:15:08,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 23:15:08,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_39000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 23:15:18,993] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 23:20:33,572] INFO : Step 39100/50000 rank7 | Train Loss: 0.7927 | Train Loss Unweighted: 0.3717 | Train Loss Components: Auxiliary: 0.1617 | Cardinality: 0.0899 | Classification: 0.0684 | Classification_binary: 0.1172 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0510 | Regression_eta: 0.0338 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0578 | LR: 2.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9856s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0584 | max_forward: 0.7954 | max_loss: 0.1416 | max_backward: 1.8331 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.1811 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0670 | Memory: rss_MB: 7143.1 | uss_MB: 6458.7 | child_rss_MB: 4548.3 | allocated_MB: 495.5 | reserved_MB: 23898.0 | inactive_split_MB: 1036.1 +rank7 [2026-10-29 23:25:22,146] INFO : Step 39200/50000 rank7 | Train Loss: 0.8131 | Train Loss Unweighted: 0.3867 | Train Loss Components: Auxiliary: 0.1741 | Cardinality: 0.0830 | Classification: 0.0716 | Classification_binary: 0.1367 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1119 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0352 | Regression_eta: 0.0305 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0514 | LR: 2.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5909s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7739 | max_loss: 0.1557 | max_backward: 1.8654 | max_optimizer_step: 0.2579 | max_tensorboard_logging: 0.8351 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5804.4 | uss_MB: 5121.3 | child_rss_MB: 4544.8 | allocated_MB: 496.4 | reserved_MB: 24172.0 | inactive_split_MB: 957.8 +rank7 [2026-10-29 23:29:56,590] INFO : Step 39300/50000 rank7 | Train Loss: 0.8921 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1302 | Classification: 0.0611 | Classification_binary: 0.1568 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0860 | Regression_eta: 0.0013 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0462 | Regression_eta: 0.0500 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0690 | LR: 1.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6297s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7996 | max_loss: 0.1852 | max_backward: 1.9004 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.8254 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 5801.4 | uss_MB: 5121.2 | child_rss_MB: 4537.0 | allocated_MB: 498.3 | reserved_MB: 25228.0 | inactive_split_MB: 1216.7 +rank7 [2026-10-29 23:34:45,046] INFO : Step 39400/50000 rank7 | Train Loss: 0.8029 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.1014 | Classification: 0.0618 | Classification_binary: 0.1177 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0886 | Regression_eta: 0.0012 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0473 | Regression_eta: 0.0528 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0915 | LR: 1.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6889s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8170 | max_loss: 0.1944 | max_backward: 2.5026 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8603 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 5803.2 | uss_MB: 4894.6 | child_rss_MB: 4534.9 | allocated_MB: 496.2 | reserved_MB: 25228.0 | inactive_split_MB: 1119.3 +rank7 [2026-10-29 23:39:27,724] INFO : Step 39500/50000 rank7 | Train Loss: 0.7876 | Train Loss Unweighted: 0.3631 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0953 | Classification: 0.0593 | Classification_binary: 0.1226 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0889 | Regression_eta: 0.0010 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0408 | Regression_eta: 0.0312 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0409 | LR: 1.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4394s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0075 | max_forward: 0.8337 | max_loss: 0.1507 | max_backward: 1.7633 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.7920 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 5800.4 | uss_MB: 5120.7 | child_rss_MB: 4537.2 | allocated_MB: 499.3 | reserved_MB: 25228.0 | inactive_split_MB: 1204.0 +rank7 [2026-10-29 23:43:57,879] INFO : Step 39600/50000 rank7 | Train Loss: 0.8291 | Train Loss Unweighted: 0.3817 | Train Loss Components: Auxiliary: 0.1797 | Cardinality: 0.1176 | Classification: 0.0696 | Classification_binary: 0.1383 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0853 | Regression_eta: 0.0011 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0402 | Regression_eta: 0.0236 | Regression_pt: 0.0414 | Regression_sin_phi: 0.0410 | LR: 1.93e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8939 | max_loss: 0.1519 | max_backward: 2.1210 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5292 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 5800.7 | uss_MB: 5121.7 | child_rss_MB: 4535.5 | allocated_MB: 498.6 | reserved_MB: 26144.0 | inactive_split_MB: 1021.1 +rank7 [2026-10-29 23:48:42,956] INFO : Step 39700/50000 rank7 | Train Loss: 0.8367 | Train Loss Unweighted: 0.3899 | Train Loss Components: Auxiliary: 0.1745 | Cardinality: 0.1049 | Classification: 0.0694 | Classification_binary: 0.1370 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0905 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0522 | Regression_eta: 0.0269 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0526 | LR: 1.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4039s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8147 | max_loss: 0.1531 | max_backward: 1.9811 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.5936 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0263 | Memory: rss_MB: 5798.6 | uss_MB: 5121.2 | child_rss_MB: 4531.3 | allocated_MB: 497.9 | reserved_MB: 28126.0 | inactive_split_MB: 970.1 +rank7 [2026-10-29 23:53:54,345] INFO : Step 39800/50000 rank7 | Train Loss: 0.7227 | Train Loss Unweighted: 0.3467 | Train Loss Components: Auxiliary: 0.1548 | Cardinality: 0.0834 | Classification: 0.0555 | Classification_binary: 0.1119 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0887 | Regression_eta: 0.0009 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.0966 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0492 | Regression_eta: 0.0227 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0345 | LR: 1.89e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5380s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 1.0173 | max_loss: 0.1680 | max_backward: 1.8334 | max_optimizer_step: 0.2880 | max_tensorboard_logging: 0.8357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5789.6 | uss_MB: 5113.6 | child_rss_MB: 4526.1 | allocated_MB: 494.7 | reserved_MB: 28126.0 | inactive_split_MB: 1010.0 +rank7 [2026-10-30 00:01:06,372] INFO : Step 39900/50000 rank7 | Train Loss: 0.8483 | Train Loss Unweighted: 0.3859 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1148 | Classification: 0.0715 | Classification_binary: 0.1371 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0869 | Regression_eta: 0.0013 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0540 | Regression_eta: 0.0297 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0692 | LR: 1.88e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7672s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0569 | max_forward: 0.7953 | max_loss: 0.1541 | max_backward: 1.8480 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 1.0140 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5792.4 | uss_MB: 5121.1 | child_rss_MB: 4490.8 | allocated_MB: 497.5 | reserved_MB: 30162.0 | inactive_split_MB: 1007.6 +rank7 [2026-10-30 00:05:55,796] INFO : Step 40000/50000 rank7 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1215 | Classification: 0.0601 | Classification_binary: 0.1491 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0962 | Regression_eta: 0.0012 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0497 | Regression_eta: 0.0349 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0605 | LR: 1.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.9282s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8237 | max_loss: 0.1984 | max_backward: 2.2096 | max_optimizer_step: 0.1923 | max_tensorboard_logging: 0.8839 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 5771.6 | uss_MB: 5116.4 | child_rss_MB: 4415.5 | allocated_MB: 499.6 | reserved_MB: 30162.0 | inactive_split_MB: 961.7 +rank7 [2026-10-30 00:05:56,736] INFO : Running validation on rank7 +rank7 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:05:56,736] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:06:50,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:06:51,352] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 00:07:17,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:07:18,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:07:19,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:07:20,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:07:30,826] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:07:31,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:07:32,019] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 00:07:47,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:07:48,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:07:49,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:07:50,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:08:00,892] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:08:01,580] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:08:02,099] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 00:08:17,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 00:08:18,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 00:08:19,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 00:08:20,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_40000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 00:08:30,132] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 00:13:50,200] INFO : Step 40100/50000 rank7 | Train Loss: 0.7849 | Train Loss Unweighted: 0.3693 | Train Loss Components: Auxiliary: 0.1620 | Cardinality: 0.0911 | Classification: 0.0629 | Classification_binary: 0.1170 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0543 | Regression_eta: 0.0278 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0421 | LR: 1.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7750 | max_loss: 0.1411 | max_backward: 1.8206 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 0.8574 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0525 | Memory: rss_MB: 7511.5 | uss_MB: 6856.3 | child_rss_MB: 4412.1 | allocated_MB: 495.1 | reserved_MB: 23134.0 | inactive_split_MB: 988.4 +rank7 [2026-10-30 00:18:43,888] INFO : Step 40200/50000 rank7 | Train Loss: 0.7641 | Train Loss Unweighted: 0.3549 | Train Loss Components: Auxiliary: 0.1605 | Cardinality: 0.0898 | Classification: 0.0648 | Classification_binary: 0.1228 | HitGrouping: 0.1385 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1266 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0474 | Regression_eta: 0.0287 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0345 | LR: 1.83e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4453s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8207 | max_loss: 0.1429 | max_backward: 1.9229 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6474 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 5901.2 | uss_MB: 5246.0 | child_rss_MB: 4410.2 | allocated_MB: 498.3 | reserved_MB: 23134.0 | inactive_split_MB: 940.7 +rank7 [2026-10-30 00:23:35,704] INFO : Step 40300/50000 rank7 | Train Loss: 0.8995 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1930 | Cardinality: 0.1080 | Classification: 0.0701 | Classification_binary: 0.1576 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0954 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0556 | Regression_eta: 0.0380 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0479 | LR: 1.81e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5361s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7948 | max_loss: 0.4899 | max_backward: 1.9606 | max_optimizer_step: 0.2784 | max_tensorboard_logging: 0.5929 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5902.4 | uss_MB: 5247.2 | child_rss_MB: 4404.2 | allocated_MB: 498.7 | reserved_MB: 23134.0 | inactive_split_MB: 1214.9 +rank7 [2026-10-30 00:28:38,369] INFO : Step 40400/50000 rank7 | Train Loss: 0.8210 | Train Loss Unweighted: 0.3621 | Train Loss Components: Auxiliary: 0.1691 | Cardinality: 0.1206 | Classification: 0.0654 | Classification_binary: 0.1271 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0830 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0426 | Regression_eta: 0.0286 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0401 | LR: 1.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2210s | Phase Times: max_pre_step_barrier: 0.0375 | max_batch_to_device: 0.0652 | max_forward: 0.8022 | max_loss: 0.1493 | max_backward: 1.7743 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.5665 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 5895.9 | uss_MB: 5242.6 | child_rss_MB: 4400.4 | allocated_MB: 498.2 | reserved_MB: 25016.0 | inactive_split_MB: 1178.6 +rank7 [2026-10-30 00:33:25,153] INFO : Step 40500/50000 rank7 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3570 | Train Loss Components: Auxiliary: 0.1618 | Cardinality: 0.1070 | Classification: 0.0670 | Classification_binary: 0.1139 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0475 | Regression_eta: 0.0328 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0495 | LR: 1.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2276s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7398 | max_loss: 0.2483 | max_backward: 1.8497 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.4765 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1277 | Memory: rss_MB: 5898.8 | uss_MB: 5245.4 | child_rss_MB: 4401.6 | allocated_MB: 496.0 | reserved_MB: 25016.0 | inactive_split_MB: 1078.3 +rank7 [2026-10-30 00:38:07,831] INFO : Step 40600/50000 rank7 | Train Loss: 0.8500 | Train Loss Unweighted: 0.4079 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.0975 | Classification: 0.0692 | Classification_binary: 0.1558 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0900 | Regression_eta: 0.0011 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0459 | Regression_eta: 0.0322 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0559 | LR: 1.76e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2962s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0069 | max_forward: 0.7690 | max_loss: 0.2450 | max_backward: 1.8053 | max_optimizer_step: 0.1954 | max_tensorboard_logging: 0.4901 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5899.0 | uss_MB: 5245.0 | child_rss_MB: 4407.7 | allocated_MB: 498.3 | reserved_MB: 25016.0 | inactive_split_MB: 1100.1 +rank7 [2026-10-30 00:42:54,583] INFO : Step 40700/50000 rank7 | Train Loss: 0.8013 | Train Loss Unweighted: 0.3712 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1199 | Classification: 0.0586 | Classification_binary: 0.1325 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0009 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0487 | Regression_eta: 0.0299 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0331 | LR: 1.75e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2535s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7600 | max_loss: 0.1703 | max_backward: 1.6772 | max_optimizer_step: 0.1973 | max_tensorboard_logging: 0.6483 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 5892.5 | uss_MB: 5243.4 | child_rss_MB: 4406.9 | allocated_MB: 497.5 | reserved_MB: 25016.0 | inactive_split_MB: 1006.9 +rank7 [2026-10-30 00:47:52,346] INFO : Step 40800/50000 rank7 | Train Loss: 0.7268 | Train Loss Unweighted: 0.3330 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0906 | Classification: 0.0583 | Classification_binary: 0.0986 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0864 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.0862 | Regression_cos_phi: 0.0209 | Regression_energy: 0.0349 | Regression_eta: 0.0198 | Regression_pt: 0.0363 | Regression_sin_phi: 0.0315 | LR: 1.73e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8890s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.7988 | max_loss: 0.1385 | max_backward: 2.0965 | max_optimizer_step: 0.1945 | max_tensorboard_logging: 0.9914 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5885.9 | uss_MB: 5237.2 | child_rss_MB: 4407.3 | allocated_MB: 497.7 | reserved_MB: 25016.0 | inactive_split_MB: 898.6 +rank7 [2026-10-30 00:52:46,795] INFO : Step 40900/50000 rank7 | Train Loss: 0.7644 | Train Loss Unweighted: 0.3595 | Train Loss Components: Auxiliary: 0.1658 | Cardinality: 0.0789 | Classification: 0.0669 | Classification_binary: 0.1272 | HitGrouping: 0.1364 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0815 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0395 | Regression_eta: 0.0356 | Regression_pt: 0.0395 | Regression_sin_phi: 0.0449 | LR: 1.72e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7066s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0066 | max_forward: 1.1475 | max_loss: 0.1648 | max_backward: 1.9753 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.7547 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 5896.5 | uss_MB: 5246.9 | child_rss_MB: 4410.9 | allocated_MB: 498.5 | reserved_MB: 25016.0 | inactive_split_MB: 1087.8 +rank7 [2026-10-30 00:57:38,928] INFO : Step 41000/50000 rank7 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3730 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0875 | Classification: 0.0590 | Classification_binary: 0.1381 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0009 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0353 | Regression_eta: 0.0227 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0453 | LR: 1.70e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3868s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0061 | max_forward: 0.8879 | max_loss: 0.1468 | max_backward: 1.9633 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 0.4552 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 5896.4 | uss_MB: 5246.7 | child_rss_MB: 4415.1 | allocated_MB: 495.2 | reserved_MB: 27068.0 | inactive_split_MB: 1095.0 +rank7 [2026-10-30 00:57:39,931] INFO : Running validation on rank7 +rank7 [2026-10-30 00:57:39,931] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:57:39,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:58:31,732] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:58:32,294] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 00:58:59,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:59:00,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:59:01,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:59:02,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:59:12,444] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:59:13,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:59:13,669] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 00:59:29,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:59:30,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:59:31,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:59:32,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:59:42,809] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:59:43,382] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:59:43,889] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 00:59:59,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:00:00,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:00:01,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:00:02,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_41000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:00:12,153] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 01:05:10,996] INFO : Step 41100/50000 rank7 | Train Loss: 0.7908 | Train Loss Unweighted: 0.3702 | Train Loss Components: Auxiliary: 0.1652 | Cardinality: 0.1084 | Classification: 0.0591 | Classification_binary: 0.1212 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0010 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0471 | Regression_eta: 0.0333 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0446 | LR: 1.69e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7656s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7983 | max_loss: 0.1433 | max_backward: 1.8474 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.9381 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 7747.3 | uss_MB: 7097.1 | child_rss_MB: 4404.6 | allocated_MB: 495.6 | reserved_MB: 24848.0 | inactive_split_MB: 998.4 +rank7 [2026-10-30 01:10:09,652] INFO : Step 41200/50000 rank7 | Train Loss: 0.7982 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0853 | Classification: 0.0627 | Classification_binary: 0.1183 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1067 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1144 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0453 | Regression_eta: 0.0378 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0369 | LR: 1.67e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6344s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.0478 | max_loss: 0.2502 | max_backward: 1.9533 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.8656 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 5938.6 | uss_MB: 5294.6 | child_rss_MB: 4387.9 | allocated_MB: 499.0 | reserved_MB: 26868.0 | inactive_split_MB: 1167.3 +rank7 [2026-10-30 01:15:00,324] INFO : Step 41300/50000 rank7 | Train Loss: 0.8568 | Train Loss Unweighted: 0.3963 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1171 | Classification: 0.0603 | Classification_binary: 0.1471 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0011 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1322 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0450 | Regression_eta: 0.0343 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0393 | LR: 1.66e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2619s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7640 | max_loss: 0.2527 | max_backward: 1.8779 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.5521 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5937.4 | uss_MB: 5295.1 | child_rss_MB: 4406.0 | allocated_MB: 495.9 | reserved_MB: 26868.0 | inactive_split_MB: 1198.0 +rank7 [2026-10-30 01:19:55,521] INFO : Step 41400/50000 rank7 | Train Loss: 0.7720 | Train Loss Unweighted: 0.3685 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0866 | Classification: 0.0645 | Classification_binary: 0.1252 | HitGrouping: 0.1439 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0878 | Regression_eta: 0.0009 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0267 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0492 | Regression_eta: 0.0293 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0510 | LR: 1.64e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.6703s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0070 | max_forward: 0.9818 | max_loss: 0.3006 | max_backward: 2.6907 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 1.0620 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5924.8 | uss_MB: 5296.2 | child_rss_MB: 4413.6 | allocated_MB: 495.8 | reserved_MB: 26868.0 | inactive_split_MB: 938.0 +rank7 [2026-10-30 01:24:55,601] INFO : Step 41500/50000 rank7 | Train Loss: 0.7008 | Train Loss Unweighted: 0.3381 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0689 | Classification: 0.0702 | Classification_binary: 0.1064 | HitGrouping: 0.1380 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0795 | Regression_eta: 0.0010 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1073 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0314 | Regression_eta: 0.0242 | Regression_pt: 0.0324 | Regression_sin_phi: 0.0537 | LR: 1.63e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.4507s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7313 | max_loss: 0.5945 | max_backward: 2.6539 | max_optimizer_step: 0.1805 | max_tensorboard_logging: 0.9593 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 5928.8 | uss_MB: 5283.4 | child_rss_MB: 4396.2 | allocated_MB: 494.7 | reserved_MB: 26868.0 | inactive_split_MB: 992.3 +rank7 [2026-10-30 01:29:39,759] INFO : Step 41600/50000 rank7 | Train Loss: 0.8229 | Train Loss Unweighted: 0.3776 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1133 | Classification: 0.0712 | Classification_binary: 0.1354 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0838 | Regression_eta: 0.0010 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1235 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0546 | Regression_eta: 0.0221 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0349 | LR: 1.61e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0148s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.9224 | max_loss: 0.1572 | max_backward: 2.1279 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.9885 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5923.2 | uss_MB: 5297.7 | child_rss_MB: 4409.7 | allocated_MB: 497.9 | reserved_MB: 28930.0 | inactive_split_MB: 1130.1 +rank7 [2026-10-30 01:34:19,969] INFO : Step 41700/50000 rank7 | Train Loss: 0.8156 | Train Loss Unweighted: 0.3714 | Train Loss Components: Auxiliary: 0.1751 | Cardinality: 0.1249 | Classification: 0.0635 | Classification_binary: 0.1389 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0831 | Regression_eta: 0.0010 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0448 | Regression_eta: 0.0323 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0346 | LR: 1.60e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7040s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7894 | max_loss: 0.2624 | max_backward: 1.9312 | max_optimizer_step: 0.2953 | max_tensorboard_logging: 0.7903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5923.2 | uss_MB: 5296.0 | child_rss_MB: 4424.7 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1249.8 +rank7 [2026-10-30 01:39:10,939] INFO : Step 41800/50000 rank7 | Train Loss: 0.7790 | Train Loss Unweighted: 0.3554 | Train Loss Components: Auxiliary: 0.1575 | Cardinality: 0.0919 | Classification: 0.0600 | Classification_binary: 0.1154 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0884 | Regression_eta: 0.0013 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0466 | Regression_eta: 0.0203 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0455 | LR: 1.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8795s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7984 | max_loss: 0.1426 | max_backward: 2.0925 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 0.9706 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5913.9 | uss_MB: 5289.9 | child_rss_MB: 4425.8 | allocated_MB: 498.6 | reserved_MB: 28930.0 | inactive_split_MB: 1106.0 +rank7 [2026-10-30 01:44:06,920] INFO : Step 41900/50000 rank7 | Train Loss: 0.7621 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1589 | Cardinality: 0.0845 | Classification: 0.0566 | Classification_binary: 0.1197 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0821 | Regression_eta: 0.0010 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0455 | Regression_eta: 0.0257 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0447 | LR: 1.57e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2478s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0457 | max_forward: 0.8097 | max_loss: 0.1454 | max_backward: 1.7510 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5828 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5922.9 | uss_MB: 5298.0 | child_rss_MB: 4423.9 | allocated_MB: 496.4 | reserved_MB: 28930.0 | inactive_split_MB: 1190.7 +rank7 [2026-10-30 01:49:02,849] INFO : Step 42000/50000 rank7 | Train Loss: 0.8378 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1132 | Classification: 0.0618 | Classification_binary: 0.1531 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0821 | Regression_eta: 0.0012 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0434 | Regression_eta: 0.0312 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0426 | LR: 1.56e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2679s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.7824 | max_loss: 0.2530 | max_backward: 1.7836 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.4819 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1131 | Memory: rss_MB: 5922.9 | uss_MB: 5297.8 | child_rss_MB: 4419.4 | allocated_MB: 497.7 | reserved_MB: 28930.0 | inactive_split_MB: 1197.1 +rank7 [2026-10-30 01:49:03,869] INFO : Running validation on rank7 +rank7 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 01:49:03,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 01:49:57,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:49:57,866] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 01:50:25,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 01:50:26,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 01:50:27,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 01:50:28,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 01:50:38,316] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-30 01:50:38,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:50:39,771] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 01:50:55,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 01:50:56,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 01:50:57,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 01:50:58,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 01:51:08,737] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:51:09,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:51:10,101] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 01:51:25,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:51:26,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:51:26,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:51:27,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_42000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:51:37,794] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 01:57:02,471] INFO : Step 42100/50000 rank7 | Train Loss: 0.7880 | Train Loss Unweighted: 0.3572 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.1079 | Classification: 0.0570 | Classification_binary: 0.1183 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0899 | Regression_eta: 0.0011 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0295 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0306 | Regression_energy: 0.0518 | Regression_eta: 0.0268 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0390 | LR: 1.54e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0033s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8024 | max_loss: 0.2553 | max_backward: 2.0527 | max_optimizer_step: 0.3147 | max_tensorboard_logging: 0.8649 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7692.5 | uss_MB: 7070.8 | child_rss_MB: 4391.8 | allocated_MB: 497.4 | reserved_MB: 27144.0 | inactive_split_MB: 1427.2 +rank7 [2026-10-30 02:01:47,469] INFO : Step 42200/50000 rank7 | Train Loss: 0.7701 | Train Loss Unweighted: 0.3543 | Train Loss Components: Auxiliary: 0.1626 | Cardinality: 0.0946 | Classification: 0.0600 | Classification_binary: 0.1239 | HitGrouping: 0.1445 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0836 | Regression_eta: 0.0011 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0267 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0359 | Regression_eta: 0.0252 | Regression_pt: 0.0369 | Regression_sin_phi: 0.0386 | LR: 1.53e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.2724s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8156 | max_loss: 0.3589 | max_backward: 3.3901 | max_optimizer_step: 0.3101 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 5963.7 | uss_MB: 4224.7 | child_rss_MB: 4389.5 | allocated_MB: 501.1 | reserved_MB: 29144.0 | inactive_split_MB: 1384.2 +rank7 [2026-10-30 02:02:38,475] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank7 [2026-10-30 02:02:38,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-30 02:06:41,298] INFO : Step 42300/50000 rank7 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0970 | Classification: 0.0675 | Classification_binary: 0.1375 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0895 | Regression_eta: 0.0010 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0501 | Regression_eta: 0.0408 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0609 | LR: 1.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8856s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9041 | max_loss: 0.1534 | max_backward: 2.0829 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.8734 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 5967.5 | uss_MB: 5363.1 | child_rss_MB: 4396.4 | allocated_MB: 500.2 | reserved_MB: 29146.0 | inactive_split_MB: 1086.2 +rank7 [2026-10-30 02:11:46,427] INFO : Step 42400/50000 rank7 | Train Loss: 0.7427 | Train Loss Unweighted: 0.3441 | Train Loss Components: Auxiliary: 0.1498 | Cardinality: 0.0783 | Classification: 0.0609 | Classification_binary: 0.1087 | HitGrouping: 0.1415 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0862 | Regression_eta: 0.0012 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0469 | Regression_eta: 0.0387 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0639 | LR: 1.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.1601s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.8040 | max_loss: 0.2184 | max_backward: 2.7899 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.5186 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.1272 | Memory: rss_MB: 5950.2 | uss_MB: 4319.1 | child_rss_MB: 4368.4 | allocated_MB: 493.2 | reserved_MB: 29146.0 | inactive_split_MB: 998.2 +rank7 [2026-10-30 02:16:38,988] INFO : Step 42500/50000 rank7 | Train Loss: 0.8433 | Train Loss Unweighted: 0.3873 | Train Loss Components: Auxiliary: 0.1798 | Cardinality: 0.1096 | Classification: 0.0681 | Classification_binary: 0.1391 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0885 | Regression_eta: 0.0011 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0436 | Regression_eta: 0.0336 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0552 | LR: 1.49e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8630s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0951 | max_forward: 0.8720 | max_loss: 0.1459 | max_backward: 1.9136 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4771 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 5942.2 | uss_MB: 5353.3 | child_rss_MB: 4386.5 | allocated_MB: 499.0 | reserved_MB: 29148.0 | inactive_split_MB: 1434.1 +rank7 [2026-10-30 02:21:27,108] INFO : Step 42600/50000 rank7 | Train Loss: 0.8723 | Train Loss Unweighted: 0.4072 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.1152 | Classification: 0.0643 | Classification_binary: 0.1610 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0893 | Regression_eta: 0.0011 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0479 | Regression_eta: 0.0387 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0503 | LR: 1.48e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.3539s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1083 | max_forward: 1.3815 | max_loss: 0.3382 | max_backward: 3.0095 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.6955 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0787 | Memory: rss_MB: 5930.5 | uss_MB: 5340.0 | child_rss_MB: 4385.8 | allocated_MB: 499.8 | reserved_MB: 29148.0 | inactive_split_MB: 1270.2 +rank7 [2026-10-30 02:26:28,199] INFO : Step 42700/50000 rank7 | Train Loss: 0.7721 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1601 | Cardinality: 0.0979 | Classification: 0.0612 | Classification_binary: 0.1122 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0951 | Regression_eta: 0.0010 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.0910 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0393 | Regression_eta: 0.0215 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0315 | LR: 1.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6229s | Phase Times: max_pre_step_barrier: 0.0013 | max_batch_to_device: 0.0061 | max_forward: 0.7381 | max_loss: 0.2427 | max_backward: 1.8894 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.9113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 5942.5 | uss_MB: 5353.1 | child_rss_MB: 4366.5 | allocated_MB: 493.9 | reserved_MB: 30072.0 | inactive_split_MB: 1420.1 +rank7 [2026-10-30 02:31:38,797] INFO : Step 42800/50000 rank7 | Train Loss: 0.7803 | Train Loss Unweighted: 0.3665 | Train Loss Components: Auxiliary: 0.1671 | Cardinality: 0.0919 | Classification: 0.0681 | Classification_binary: 0.1234 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0856 | Regression_eta: 0.0010 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0514 | Regression_eta: 0.0550 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0726 | LR: 1.45e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6419s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7829 | max_loss: 0.1489 | max_backward: 1.9135 | max_optimizer_step: 0.1977 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0885 | Memory: rss_MB: 5942.5 | uss_MB: 5355.0 | child_rss_MB: 4386.8 | allocated_MB: 498.9 | reserved_MB: 30072.0 | inactive_split_MB: 961.2 +rank7 [2026-10-30 02:36:26,141] INFO : Step 42900/50000 rank7 | Train Loss: 0.8162 | Train Loss Unweighted: 0.3701 | Train Loss Components: Auxiliary: 0.1760 | Cardinality: 0.1155 | Classification: 0.0582 | Classification_binary: 0.1462 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0817 | Regression_eta: 0.0010 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0334 | Regression_energy: 0.0420 | Regression_eta: 0.0214 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0424 | LR: 1.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8530s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.9422 | max_loss: 0.1533 | max_backward: 2.1906 | max_optimizer_step: 0.1994 | max_tensorboard_logging: 0.7770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 5938.5 | uss_MB: 5351.5 | child_rss_MB: 4384.7 | allocated_MB: 501.1 | reserved_MB: 31184.0 | inactive_split_MB: 954.8 +rank7 [2026-10-30 02:41:13,872] INFO : Step 43000/50000 rank7 | Train Loss: 0.7601 | Train Loss Unweighted: 0.3546 | Train Loss Components: Auxiliary: 0.1604 | Cardinality: 0.0866 | Classification: 0.0654 | Classification_binary: 0.1248 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0809 | Regression_eta: 0.0010 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0335 | Regression_eta: 0.0297 | Regression_pt: 0.0352 | Regression_sin_phi: 0.0519 | LR: 1.43e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5958s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7779 | max_loss: 0.1522 | max_backward: 1.9216 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.7083 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1314 | Memory: rss_MB: 5926.5 | uss_MB: 5351.0 | child_rss_MB: 4375.1 | allocated_MB: 499.7 | reserved_MB: 31184.0 | inactive_split_MB: 1180.3 +rank7 [2026-10-30 02:41:14,879] INFO : Running validation on rank7 +rank7 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 02:41:14,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 02:42:10,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:42:10,783] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 02:42:38,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 02:42:39,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 02:42:40,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 02:42:41,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 02:42:51,252] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 02:42:51,954] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:42:52,442] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 02:43:09,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 02:43:10,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 02:43:10,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 02:43:11,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 02:43:22,002] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 02:43:22,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:43:23,032] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 02:43:38,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 02:43:39,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 02:43:40,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 02:43:41,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_43000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 02:43:51,469] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 02:48:51,375] INFO : Step 43100/50000 rank7 | Train Loss: 0.7313 | Train Loss Unweighted: 0.3362 | Train Loss Components: Auxiliary: 0.1519 | Cardinality: 0.0850 | Classification: 0.0590 | Classification_binary: 0.1121 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0814 | Regression_eta: 0.0010 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.0951 | Regression_cos_phi: 0.0335 | Regression_energy: 0.0352 | Regression_eta: 0.0232 | Regression_pt: 0.0361 | Regression_sin_phi: 0.0334 | LR: 1.42e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9463s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7706 | max_loss: 0.2117 | max_backward: 1.7925 | max_optimizer_step: 0.3346 | max_tensorboard_logging: 0.9899 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 7814.3 | uss_MB: 7237.4 | child_rss_MB: 4395.9 | allocated_MB: 495.6 | reserved_MB: 27372.0 | inactive_split_MB: 1091.6 +rank7 [2026-10-30 02:53:29,172] INFO : Step 43200/50000 rank7 | Train Loss: 0.8455 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1859 | Cardinality: 0.1014 | Classification: 0.0751 | Classification_binary: 0.1428 | HitGrouping: 0.1373 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0938 | Regression_eta: 0.0012 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0415 | Regression_eta: 0.0270 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0520 | LR: 1.40e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5643s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8547 | max_loss: 0.1499 | max_backward: 1.9316 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.7031 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5993.3 | uss_MB: 5417.1 | child_rss_MB: 4384.1 | allocated_MB: 499.3 | reserved_MB: 31286.0 | inactive_split_MB: 1380.7 +rank7 [2026-10-30 02:58:07,501] INFO : Step 43300/50000 rank7 | Train Loss: 0.8333 | Train Loss Unweighted: 0.3764 | Train Loss Components: Auxiliary: 0.1795 | Cardinality: 0.1203 | Classification: 0.0597 | Classification_binary: 0.1360 | HitGrouping: 0.1408 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0011 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0494 | Regression_eta: 0.0348 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0487 | LR: 1.39e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0490s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.9566 | max_loss: 0.1535 | max_backward: 2.2307 | max_optimizer_step: 0.2026 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 5984.3 | uss_MB: 5419.9 | child_rss_MB: 4383.2 | allocated_MB: 499.0 | reserved_MB: 33322.0 | inactive_split_MB: 1293.0 +rank7 [2026-10-30 03:02:42,807] INFO : Step 43400/50000 rank7 | Train Loss: 0.7171 | Train Loss Unweighted: 0.3399 | Train Loss Components: Auxiliary: 0.1486 | Cardinality: 0.0813 | Classification: 0.0602 | Classification_binary: 0.1063 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0860 | Regression_eta: 0.0009 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0428 | Regression_eta: 0.0303 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0319 | LR: 1.38e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4732s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7206 | max_loss: 0.1501 | max_backward: 1.6984 | max_optimizer_step: 0.2996 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 5983.0 | uss_MB: 5417.7 | child_rss_MB: 4378.5 | allocated_MB: 493.5 | reserved_MB: 33322.0 | inactive_split_MB: 1419.6 +rank7 [2026-10-30 03:07:16,795] INFO : Step 43500/50000 rank7 | Train Loss: 0.7562 | Train Loss Unweighted: 0.3579 | Train Loss Components: Auxiliary: 0.1663 | Cardinality: 0.0805 | Classification: 0.0610 | Classification_binary: 0.1308 | HitGrouping: 0.1369 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0817 | Regression_eta: 0.0011 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0500 | Regression_eta: 0.0317 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0357 | LR: 1.37e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9332s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7810 | max_loss: 0.1733 | max_backward: 1.8998 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 5984.0 | uss_MB: 5419.8 | child_rss_MB: 4386.8 | allocated_MB: 497.7 | reserved_MB: 33322.0 | inactive_split_MB: 1201.2 +rank7 [2026-10-30 03:12:00,122] INFO : Step 43600/50000 rank7 | Train Loss: 0.8935 | Train Loss Unweighted: 0.3960 | Train Loss Components: Auxiliary: 0.1873 | Cardinality: 0.1160 | Classification: 0.0669 | Classification_binary: 0.1563 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0847 | Regression_eta: 0.0013 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0419 | Regression_eta: 0.0525 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0488 | LR: 1.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8420s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0074 | max_forward: 0.8010 | max_loss: 0.2622 | max_backward: 1.9528 | max_optimizer_step: 0.2685 | max_tensorboard_logging: 0.8957 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0301 | Memory: rss_MB: 5968.3 | uss_MB: 5407.2 | child_rss_MB: 4384.7 | allocated_MB: 498.1 | reserved_MB: 33324.0 | inactive_split_MB: 952.6 +rank7 [2026-10-30 03:16:33,742] INFO : Step 43700/50000 rank7 | Train Loss: 0.7845 | Train Loss Unweighted: 0.3625 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0949 | Classification: 0.0613 | Classification_binary: 0.1176 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0906 | Regression_eta: 0.0011 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1122 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0476 | Regression_eta: 0.0265 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0498 | LR: 1.35e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1349s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7144 | max_loss: 0.1428 | max_backward: 1.7336 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5645 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5980.0 | uss_MB: 5417.3 | child_rss_MB: 4399.0 | allocated_MB: 495.0 | reserved_MB: 33326.0 | inactive_split_MB: 1001.0 +rank7 [2026-10-30 03:21:10,514] INFO : Step 43800/50000 rank7 | Train Loss: 0.7763 | Train Loss Unweighted: 0.3669 | Train Loss Components: Auxiliary: 0.1645 | Cardinality: 0.0864 | Classification: 0.0637 | Classification_binary: 0.1175 | HitGrouping: 0.1442 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0917 | Regression_eta: 0.0011 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1010 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0405 | Regression_eta: 0.0358 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0424 | LR: 1.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2537s | Phase Times: max_pre_step_barrier: 0.1116 | max_batch_to_device: 0.0065 | max_forward: 0.7253 | max_loss: 0.2350 | max_backward: 1.7924 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.4929 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0333 | Memory: rss_MB: 5976.0 | uss_MB: 5418.6 | child_rss_MB: 4386.4 | allocated_MB: 498.0 | reserved_MB: 33326.0 | inactive_split_MB: 1130.0 +rank7 [2026-10-30 03:26:01,000] INFO : Step 43900/50000 rank7 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0926 | Classification: 0.0703 | Classification_binary: 0.1513 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0871 | Regression_eta: 0.0014 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0422 | Regression_eta: 0.0255 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0431 | LR: 1.33e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9689s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7858 | max_loss: 0.2406 | max_backward: 1.8832 | max_optimizer_step: 0.2818 | max_tensorboard_logging: 1.0583 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 5974.5 | uss_MB: 5416.0 | child_rss_MB: 4392.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1279.3 +rank7 [2026-10-30 03:30:36,933] INFO : Step 44000/50000 rank7 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1659 | Cardinality: 0.1014 | Classification: 0.0588 | Classification_binary: 0.1330 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1318 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0492 | Regression_eta: 0.0301 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0458 | LR: 1.32e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8184s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1020 | max_forward: 0.8503 | max_loss: 0.1531 | max_backward: 1.9950 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.8952 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5976.5 | uss_MB: 5417.9 | child_rss_MB: 4386.1 | allocated_MB: 498.7 | reserved_MB: 33326.0 | inactive_split_MB: 1106.7 +rank7 [2026-10-30 03:30:37,938] INFO : Running validation on rank7 +rank7 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 03:30:37,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 03:31:28,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:31:29,164] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 03:31:57,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 03:31:58,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 03:31:59,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 03:32:00,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 03:32:10,763] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 03:32:11,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:32:11,903] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 03:32:27,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 03:32:28,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 03:32:29,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 03:32:30,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 03:32:40,534] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 03:32:41,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:32:41,627] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 03:32:56,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 03:32:57,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 03:32:58,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 03:32:59,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_44000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 03:33:09,252] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 03:38:15,294] INFO : Step 44100/50000 rank7 | Train Loss: 0.7488 | Train Loss Unweighted: 0.3424 | Train Loss Components: Auxiliary: 0.1566 | Cardinality: 0.0942 | Classification: 0.0598 | Classification_binary: 0.1078 | HitGrouping: 0.1467 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0266 | Classification_binary: 0.1015 | Regression_cos_phi: 0.0301 | Regression_energy: 0.0460 | Regression_eta: 0.0271 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0458 | LR: 1.31e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7488s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.7761 | max_loss: 0.2333 | max_backward: 1.8758 | max_optimizer_step: 0.3203 | max_tensorboard_logging: 0.7995 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0650 | Memory: rss_MB: 7771.0 | uss_MB: 7206.8 | child_rss_MB: 4386.8 | allocated_MB: 494.5 | reserved_MB: 24854.0 | inactive_split_MB: 1221.6 +rank7 [2026-10-30 03:43:04,521] INFO : Step 44200/50000 rank7 | Train Loss: 0.8143 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.1159 | Classification: 0.0629 | Classification_binary: 0.1348 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0847 | Regression_eta: 0.0010 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0464 | Regression_eta: 0.0195 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0384 | LR: 1.30e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0066 | max_forward: 0.7732 | max_loss: 0.1928 | max_backward: 1.8172 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.5863 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6052.8 | uss_MB: 5493.6 | child_rss_MB: 4413.7 | allocated_MB: 499.3 | reserved_MB: 26324.0 | inactive_split_MB: 1060.7 +rank7 [2026-10-30 03:47:46,464] INFO : Step 44300/50000 rank7 | Train Loss: 0.8071 | Train Loss Unweighted: 0.3649 | Train Loss Components: Auxiliary: 0.1715 | Cardinality: 0.1017 | Classification: 0.0608 | Classification_binary: 0.1357 | HitGrouping: 0.1390 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0824 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0401 | Regression_eta: 0.0232 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0368 | LR: 1.29e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4317s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.8201 | max_loss: 0.1610 | max_backward: 1.9322 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.5663 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6038.2 | uss_MB: 5494.7 | child_rss_MB: 4326.6 | allocated_MB: 498.1 | reserved_MB: 26326.0 | inactive_split_MB: 1103.9 +rank7 [2026-10-30 03:52:24,792] INFO : Step 44400/50000 rank7 | Train Loss: 0.7150 | Train Loss Unweighted: 0.3396 | Train Loss Components: Auxiliary: 0.1493 | Cardinality: 0.0835 | Classification: 0.0587 | Classification_binary: 0.1042 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0872 | Regression_eta: 0.0009 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0204 | Classification_binary: 0.0888 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0358 | Regression_eta: 0.0193 | Regression_pt: 0.0382 | Regression_sin_phi: 0.0466 | LR: 1.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3365s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0634 | max_forward: 0.7948 | max_loss: 0.1477 | max_backward: 1.8477 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.5705 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6023.5 | uss_MB: 5480.2 | child_rss_MB: 4329.2 | allocated_MB: 494.4 | reserved_MB: 26326.0 | inactive_split_MB: 1217.6 +rank7 [2026-10-30 03:57:11,971] INFO : Step 44500/50000 rank7 | Train Loss: 0.7740 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1623 | Cardinality: 0.0898 | Classification: 0.0627 | Classification_binary: 0.1243 | HitGrouping: 0.1392 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0820 | Regression_eta: 0.0011 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0508 | Regression_eta: 0.0305 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0543 | LR: 1.27e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3164s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 0.7688 | max_loss: 0.1458 | max_backward: 1.9755 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.4882 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6033.0 | uss_MB: 5492.8 | child_rss_MB: 4321.9 | allocated_MB: 497.8 | reserved_MB: 26326.0 | inactive_split_MB: 1215.5 +rank7 [2026-10-30 04:01:48,197] INFO : Step 44600/50000 rank7 | Train Loss: 0.8709 | Train Loss Unweighted: 0.3918 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1169 | Classification: 0.0658 | Classification_binary: 0.1472 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0428 | Regression_eta: 0.0306 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0561 | LR: 1.26e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5944s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8183 | max_loss: 0.1486 | max_backward: 2.0263 | max_optimizer_step: 0.2021 | max_tensorboard_logging: 0.5855 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0992 | Memory: rss_MB: 6008.3 | uss_MB: 5483.0 | child_rss_MB: 4279.8 | allocated_MB: 497.9 | reserved_MB: 26332.0 | inactive_split_MB: 1058.1 +rank7 [2026-10-30 04:06:28,269] INFO : Step 44700/50000 rank7 | Train Loss: 0.7087 | Train Loss Unweighted: 0.3401 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0803 | Classification: 0.0506 | Classification_binary: 0.1098 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0886 | Regression_eta: 0.0008 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0226 | Classification_binary: 0.0942 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0351 | Regression_eta: 0.0293 | Regression_pt: 0.0368 | Regression_sin_phi: 0.0298 | LR: 1.25e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6224s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0111 | max_forward: 0.8333 | max_loss: 0.1795 | max_backward: 1.9155 | max_optimizer_step: 0.2864 | max_tensorboard_logging: 0.7462 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6018.3 | uss_MB: 5491.8 | child_rss_MB: 4256.3 | allocated_MB: 492.8 | reserved_MB: 26878.0 | inactive_split_MB: 1013.9 +rank7 [2026-10-30 04:11:07,219] INFO : Step 44800/50000 rank7 | Train Loss: 0.7176 | Train Loss Unweighted: 0.3335 | Train Loss Components: Auxiliary: 0.1523 | Cardinality: 0.0858 | Classification: 0.0613 | Classification_binary: 0.1086 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0009 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0263 | Classification_binary: 0.0865 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0384 | Regression_eta: 0.0235 | Regression_pt: 0.0397 | Regression_sin_phi: 0.0552 | LR: 1.24e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.5853s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0069 | max_forward: 0.9191 | max_loss: 0.3168 | max_backward: 1.8667 | max_optimizer_step: 0.2028 | max_tensorboard_logging: 0.8431 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6017.0 | uss_MB: 5493.3 | child_rss_MB: 4271.4 | allocated_MB: 497.7 | reserved_MB: 26878.0 | inactive_split_MB: 1182.2 +rank7 [2026-10-30 04:15:54,538] INFO : Step 44900/50000 rank7 | Train Loss: 0.7932 | Train Loss Unweighted: 0.3661 | Train Loss Components: Auxiliary: 0.1764 | Cardinality: 0.1004 | Classification: 0.0595 | Classification_binary: 0.1316 | HitGrouping: 0.1368 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0860 | Regression_eta: 0.0010 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0436 | Regression_eta: 0.0345 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0485 | LR: 1.23e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6733s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0075 | max_forward: 0.7942 | max_loss: 0.1634 | max_backward: 1.9812 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 0.8032 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0331 | Memory: rss_MB: 6014.9 | uss_MB: 5494.6 | child_rss_MB: 4243.3 | allocated_MB: 501.8 | reserved_MB: 28348.0 | inactive_split_MB: 1303.0 +rank7 [2026-10-30 04:20:36,056] INFO : Step 45000/50000 rank7 | Train Loss: 0.8448 | Train Loss Unweighted: 0.3901 | Train Loss Components: Auxiliary: 0.1792 | Cardinality: 0.1189 | Classification: 0.0634 | Classification_binary: 0.1410 | HitGrouping: 0.1411 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0917 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0580 | Regression_eta: 0.0262 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0376 | LR: 1.22e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7895s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8985 | max_loss: 0.1567 | max_backward: 2.0485 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.7735 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6011.3 | uss_MB: 5170.6 | child_rss_MB: 4234.9 | allocated_MB: 495.2 | reserved_MB: 28348.0 | inactive_split_MB: 944.7 +rank7 [2026-10-30 04:20:37,109] INFO : Running validation on rank7 +rank7 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 04:20:37,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 04:21:27,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:21:28,562] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 04:21:56,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 04:21:57,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 04:21:58,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 04:21:59,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 04:22:09,956] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-30 04:22:10,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:22:10,891] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 04:22:27,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 04:22:28,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 04:22:29,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 04:22:30,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 04:22:40,124] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 04:22:40,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:22:41,055] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 04:22:56,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 04:22:57,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 04:22:57,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 04:22:58,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_45000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 04:23:08,740] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 04:28:04,849] INFO : Step 45100/50000 rank7 | Train Loss: 0.7452 | Train Loss Unweighted: 0.3499 | Train Loss Components: Auxiliary: 0.1557 | Cardinality: 0.0874 | Classification: 0.0606 | Classification_binary: 0.1053 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0908 | Regression_eta: 0.0012 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1079 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0400 | Regression_eta: 0.0266 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0377 | LR: 1.21e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2095s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0293 | max_forward: 0.8335 | max_loss: 0.1364 | max_backward: 1.6448 | max_optimizer_step: 0.2358 | max_tensorboard_logging: 0.5889 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0671 | Memory: rss_MB: 7924.1 | uss_MB: 7402.1 | child_rss_MB: 4238.4 | allocated_MB: 497.8 | reserved_MB: 28422.0 | inactive_split_MB: 1202.5 +rank7 [2026-10-30 04:32:57,431] INFO : Step 45200/50000 rank7 | Train Loss: 0.7847 | Train Loss Unweighted: 0.3697 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0937 | Classification: 0.0706 | Classification_binary: 0.1311 | HitGrouping: 0.1378 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0825 | Regression_eta: 0.0011 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0484 | Regression_eta: 0.0318 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0388 | LR: 1.20e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4466s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.8526 | max_loss: 0.1522 | max_backward: 1.6538 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 0.7446 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6102.2 | uss_MB: 5583.1 | child_rss_MB: 4228.8 | allocated_MB: 498.0 | reserved_MB: 30386.0 | inactive_split_MB: 988.5 +rank7 [2026-10-30 04:37:52,085] INFO : Step 45300/50000 rank7 | Train Loss: 0.8202 | Train Loss Unweighted: 0.3822 | Train Loss Components: Auxiliary: 0.1796 | Cardinality: 0.1118 | Classification: 0.0688 | Classification_binary: 0.1416 | HitGrouping: 0.1400 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0843 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0479 | Regression_eta: 0.0365 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0379 | LR: 1.19e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8148s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8767 | max_loss: 0.1660 | max_backward: 2.0124 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1104 | Memory: rss_MB: 6102.2 | uss_MB: 5585.2 | child_rss_MB: 4229.7 | allocated_MB: 503.1 | reserved_MB: 32390.0 | inactive_split_MB: 1202.5 +rank7 [2026-10-30 04:42:38,426] INFO : Step 45400/50000 rank7 | Train Loss: 0.7209 | Train Loss Unweighted: 0.3537 | Train Loss Components: Auxiliary: 0.1551 | Cardinality: 0.0762 | Classification: 0.0547 | Classification_binary: 0.1110 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0924 | Regression_eta: 0.0010 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0250 | Classification_binary: 0.1092 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0414 | Regression_eta: 0.0181 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0315 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4014s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7661 | max_loss: 0.1427 | max_backward: 1.5654 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.8229 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 6091.1 | uss_MB: 5573.5 | child_rss_MB: 4231.4 | allocated_MB: 497.9 | reserved_MB: 32390.0 | inactive_split_MB: 1207.0 +rank7 [2026-10-30 04:47:26,383] INFO : Step 45500/50000 rank7 | Train Loss: 0.7362 | Train Loss Unweighted: 0.3428 | Train Loss Components: Auxiliary: 0.1586 | Cardinality: 0.0820 | Classification: 0.0578 | Classification_binary: 0.1171 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0010 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0238 | Classification_binary: 0.1127 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0467 | Regression_eta: 0.0275 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0709 | LR: 1.18e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3862s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7976 | max_loss: 0.1461 | max_backward: 1.8749 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.5968 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6102.1 | uss_MB: 5587.2 | child_rss_MB: 4228.5 | allocated_MB: 497.6 | reserved_MB: 32390.0 | inactive_split_MB: 1105.5 +rank7 [2026-10-30 04:52:16,191] INFO : Step 45600/50000 rank7 | Train Loss: 0.8301 | Train Loss Unweighted: 0.3800 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.1080 | Classification: 0.0648 | Classification_binary: 0.1446 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0841 | Regression_eta: 0.0012 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0453 | Regression_eta: 0.0299 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0396 | LR: 1.17e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4804s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0062 | max_forward: 0.8239 | max_loss: 0.5065 | max_backward: 2.0484 | max_optimizer_step: 0.2020 | max_tensorboard_logging: 0.5090 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6103.1 | uss_MB: 5589.4 | child_rss_MB: 4233.6 | allocated_MB: 498.5 | reserved_MB: 32390.0 | inactive_split_MB: 901.7 +rank7 [2026-10-30 04:57:07,844] INFO : Step 45700/50000 rank7 | Train Loss: 0.7647 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1619 | Cardinality: 0.0857 | Classification: 0.0633 | Classification_binary: 0.1235 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0813 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0437 | Regression_eta: 0.0392 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0485 | LR: 1.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4878s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7902 | max_loss: 0.1534 | max_backward: 1.7735 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.6817 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6098.6 | uss_MB: 5585.2 | child_rss_MB: 4232.6 | allocated_MB: 497.5 | reserved_MB: 32394.0 | inactive_split_MB: 1390.5 +rank7 [2026-10-30 05:01:56,081] INFO : Step 45800/50000 rank7 | Train Loss: 0.7231 | Train Loss Unweighted: 0.3530 | Train Loss Components: Auxiliary: 0.1516 | Cardinality: 0.0761 | Classification: 0.0612 | Classification_binary: 0.1056 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0918 | Regression_eta: 0.0010 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0232 | Classification_binary: 0.0820 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0378 | Regression_eta: 0.0220 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0270 | LR: 1.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6464s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7840 | max_loss: 0.1493 | max_backward: 2.0333 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7033 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6101.8 | uss_MB: 5589.5 | child_rss_MB: 4238.6 | allocated_MB: 497.6 | reserved_MB: 34430.0 | inactive_split_MB: 1005.4 +rank7 [2026-10-30 05:06:42,061] INFO : Step 45900/50000 rank7 | Train Loss: 0.7345 | Train Loss Unweighted: 0.3460 | Train Loss Components: Auxiliary: 0.1598 | Cardinality: 0.0802 | Classification: 0.0577 | Classification_binary: 0.1261 | HitGrouping: 0.1341 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0803 | Regression_eta: 0.0011 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0449 | Regression_eta: 0.0267 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0398 | LR: 1.15e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2871s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0431 | max_forward: 0.8045 | max_loss: 0.1447 | max_backward: 1.8527 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6098.1 | uss_MB: 5584.7 | child_rss_MB: 4230.6 | allocated_MB: 498.3 | reserved_MB: 36518.0 | inactive_split_MB: 1122.8 +rank7 [2026-10-30 05:11:36,041] INFO : Step 46000/50000 rank7 | Train Loss: 0.7614 | Train Loss Unweighted: 0.3547 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0964 | Classification: 0.0549 | Classification_binary: 0.1301 | HitGrouping: 0.1374 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0837 | Regression_eta: 0.0009 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0400 | Regression_eta: 0.0309 | Regression_pt: 0.0403 | Regression_sin_phi: 0.0341 | LR: 1.14e-04 | DataLoad Time: 0.0225s | Model Forward Time: 6.4324s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0065 | max_forward: 0.7875 | max_loss: 0.1957 | max_backward: 1.8500 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 2.5126 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6096.1 | uss_MB: 5586.0 | child_rss_MB: 4220.9 | allocated_MB: 496.3 | reserved_MB: 36518.0 | inactive_split_MB: 1260.0 +rank7 [2026-10-30 05:11:37,038] INFO : Running validation on rank7 +rank7 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 05:11:37,038] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 05:12:27,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:12:27,993] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 05:12:52,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 05:12:53,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 05:12:54,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 05:12:55,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 05:13:05,891] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 05:13:06,687] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:13:07,159] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 05:13:23,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 05:13:24,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 05:13:24,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 05:13:25,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 05:13:35,942] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 05:13:36,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:13:37,282] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 05:13:52,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 05:13:53,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 05:13:54,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 05:13:54,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_46000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 05:14:04,884] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 05:19:00,536] INFO : Step 46100/50000 rank7 | Train Loss: 0.7265 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1556 | Cardinality: 0.0694 | Classification: 0.0626 | Classification_binary: 0.1060 | HitGrouping: 0.1448 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0953 | Regression_eta: 0.0009 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1072 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0677 | Regression_eta: 0.0342 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0591 | LR: 1.13e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5893s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0057 | max_forward: 0.7991 | max_loss: 0.2160 | max_backward: 1.9085 | max_optimizer_step: 0.3099 | max_tensorboard_logging: 0.5943 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7899.4 | uss_MB: 7383.3 | child_rss_MB: 4224.0 | allocated_MB: 495.5 | reserved_MB: 25110.0 | inactive_split_MB: 881.1 +rank7 [2026-10-30 05:23:34,445] INFO : Step 46200/50000 rank7 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3656 | Train Loss Components: Auxiliary: 0.1704 | Cardinality: 0.0813 | Classification: 0.0632 | Classification_binary: 0.1298 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0846 | Regression_eta: 0.0010 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0396 | Regression_eta: 0.0352 | Regression_pt: 0.0408 | Regression_sin_phi: 0.0408 | LR: 1.13e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7916s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7896 | max_loss: 0.2351 | max_backward: 1.9351 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0377 | Memory: rss_MB: 6168.3 | uss_MB: 5653.8 | child_rss_MB: 4222.8 | allocated_MB: 495.7 | reserved_MB: 25110.0 | inactive_split_MB: 960.5 +rank7 [2026-10-30 05:28:08,608] INFO : Step 46300/50000 rank7 | Train Loss: 0.8059 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1791 | Cardinality: 0.0912 | Classification: 0.0560 | Classification_binary: 0.1400 | HitGrouping: 0.1396 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0452 | Regression_eta: 0.0355 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0384 | LR: 1.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9762s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.9311 | max_loss: 0.1534 | max_backward: 2.1909 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6990 | max_tensorboard_flush: 0.0052 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6175.8 | uss_MB: 5663.7 | child_rss_MB: 4217.7 | allocated_MB: 498.1 | reserved_MB: 25110.0 | inactive_split_MB: 972.6 +rank7 [2026-10-30 05:32:46,648] INFO : Step 46400/50000 rank7 | Train Loss: 0.7739 | Train Loss Unweighted: 0.3557 | Train Loss Components: Auxiliary: 0.1610 | Cardinality: 0.0904 | Classification: 0.0576 | Classification_binary: 0.1199 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0874 | Regression_eta: 0.0010 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1137 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0469 | Regression_eta: 0.0325 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0408 | LR: 1.11e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7894s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7740 | max_loss: 0.2517 | max_backward: 1.8730 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.9994 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6165.7 | uss_MB: 5653.1 | child_rss_MB: 4222.0 | allocated_MB: 497.7 | reserved_MB: 25110.0 | inactive_split_MB: 926.0 +rank7 [2026-10-30 05:37:21,831] INFO : Step 46500/50000 rank7 | Train Loss: 0.7497 | Train Loss Unweighted: 0.3482 | Train Loss Components: Auxiliary: 0.1592 | Cardinality: 0.0899 | Classification: 0.0663 | Classification_binary: 0.1092 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0850 | Regression_eta: 0.0011 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0289 | Regression_energy: 0.0496 | Regression_eta: 0.0213 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0378 | LR: 1.11e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8413s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0070 | max_forward: 0.7925 | max_loss: 0.2491 | max_backward: 2.1706 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.8428 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6172.3 | uss_MB: 5663.1 | child_rss_MB: 4215.0 | allocated_MB: 497.7 | reserved_MB: 26128.0 | inactive_split_MB: 1332.5 +rank7 [2026-10-30 05:41:57,343] INFO : Step 46600/50000 rank7 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3759 | Train Loss Components: Auxiliary: 0.1771 | Cardinality: 0.1066 | Classification: 0.0652 | Classification_binary: 0.1361 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0010 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0431 | Regression_eta: 0.0263 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0395 | LR: 1.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7847s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.1012 | max_forward: 0.8809 | max_loss: 0.1532 | max_backward: 1.8499 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.8564 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6170.7 | uss_MB: 5662.1 | child_rss_MB: 4217.0 | allocated_MB: 498.9 | reserved_MB: 28164.0 | inactive_split_MB: 955.3 +rank7 [2026-10-30 05:46:32,411] INFO : Step 46700/50000 rank7 | Train Loss: 0.7909 | Train Loss Unweighted: 0.3779 | Train Loss Components: Auxiliary: 0.1729 | Cardinality: 0.0949 | Classification: 0.0592 | Classification_binary: 0.1242 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0010 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0251 | Classification_binary: 0.1185 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0337 | Regression_eta: 0.0234 | Regression_pt: 0.0346 | Regression_sin_phi: 0.0417 | LR: 1.10e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6536s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0072 | max_forward: 0.7967 | max_loss: 0.1474 | max_backward: 1.8962 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.7935 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 6170.2 | uss_MB: 5660.9 | child_rss_MB: 4222.6 | allocated_MB: 500.5 | reserved_MB: 28164.0 | inactive_split_MB: 963.5 +rank7 [2026-10-30 05:51:05,142] INFO : Step 46800/50000 rank7 | Train Loss: 0.7165 | Train Loss Unweighted: 0.3267 | Train Loss Components: Auxiliary: 0.1490 | Cardinality: 0.0760 | Classification: 0.0576 | Classification_binary: 0.1097 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0785 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0298 | Classification_binary: 0.1066 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0408 | Regression_eta: 0.0307 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0504 | LR: 1.09e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7014s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.7853 | max_loss: 0.5156 | max_backward: 1.9962 | max_optimizer_step: 0.2399 | max_tensorboard_logging: 0.7754 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6174.2 | uss_MB: 5667.1 | child_rss_MB: 4220.6 | allocated_MB: 496.0 | reserved_MB: 28164.0 | inactive_split_MB: 971.0 +rank7 [2026-10-30 05:55:41,597] INFO : Step 46900/50000 rank7 | Train Loss: 0.8043 | Train Loss Unweighted: 0.3691 | Train Loss Components: Auxiliary: 0.1762 | Cardinality: 0.1018 | Classification: 0.0653 | Classification_binary: 0.1363 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0826 | Regression_eta: 0.0011 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1078 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0480 | Regression_eta: 0.0249 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0518 | LR: 1.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7522s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8437 | max_loss: 0.1484 | max_backward: 1.9640 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.7930 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6165.2 | uss_MB: 5662.1 | child_rss_MB: 4218.2 | allocated_MB: 497.7 | reserved_MB: 28164.0 | inactive_split_MB: 927.3 +rank7 [2026-10-30 06:00:20,616] INFO : Step 47000/50000 rank7 | Train Loss: 0.8023 | Train Loss Unweighted: 0.3632 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.1012 | Classification: 0.0580 | Classification_binary: 0.1381 | HitGrouping: 0.1398 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0824 | Regression_eta: 0.0012 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0273 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0262 | Regression_energy: 0.0387 | Regression_eta: 0.0336 | Regression_pt: 0.0399 | Regression_sin_phi: 0.0494 | LR: 1.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8061s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8280 | max_loss: 0.2247 | max_backward: 2.1933 | max_optimizer_step: 0.2091 | max_tensorboard_logging: 0.6831 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6167.2 | uss_MB: 5664.7 | child_rss_MB: 4214.4 | allocated_MB: 498.3 | reserved_MB: 28166.0 | inactive_split_MB: 863.8 +rank7 [2026-10-30 06:00:21,750] INFO : Running validation on rank7 +rank7 [2026-10-30 06:00:21,751] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 06:00:21,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 06:01:12,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:01:12,842] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 06:01:37,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 06:01:38,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 06:01:39,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 06:01:40,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 06:01:50,372] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:01:51,183] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:01:51,692] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 06:02:07,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 06:02:08,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 06:02:09,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 06:02:10,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 06:02:20,637] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:02:21,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:02:21,871] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 06:02:37,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 06:02:38,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 06:02:39,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 06:02:40,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_47000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 06:02:50,114] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 06:07:51,194] INFO : Step 47100/50000 rank7 | Train Loss: 0.7309 | Train Loss Unweighted: 0.3273 | Train Loss Components: Auxiliary: 0.1502 | Cardinality: 0.0798 | Classification: 0.0600 | Classification_binary: 0.1110 | HitGrouping: 0.1391 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0768 | Regression_eta: 0.0012 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0264 | Classification_binary: 0.1020 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0447 | Regression_eta: 0.0316 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0428 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6249s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0563 | max_forward: 0.8264 | max_loss: 0.1444 | max_backward: 1.7493 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7938 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0610 | Memory: rss_MB: 7925.4 | uss_MB: 7415.1 | child_rss_MB: 4185.2 | allocated_MB: 496.0 | reserved_MB: 24066.0 | inactive_split_MB: 1198.8 +rank7 [2026-10-30 06:12:29,636] INFO : Step 47200/50000 rank7 | Train Loss: 0.7232 | Train Loss Unweighted: 0.3311 | Train Loss Components: Auxiliary: 0.1527 | Cardinality: 0.0792 | Classification: 0.0618 | Classification_binary: 0.1182 | HitGrouping: 0.1357 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0739 | Regression_eta: 0.0011 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1120 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0358 | Regression_eta: 0.0319 | Regression_pt: 0.0381 | Regression_sin_phi: 0.0375 | LR: 1.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9888s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.8733 | max_loss: 0.1495 | max_backward: 1.8594 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 1.1132 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0506 | Memory: rss_MB: 6255.8 | uss_MB: 5747.3 | child_rss_MB: 4186.8 | allocated_MB: 500.2 | reserved_MB: 26074.0 | inactive_split_MB: 1061.4 +rank7 [2026-10-30 06:17:04,152] INFO : Step 47300/50000 rank7 | Train Loss: 0.8293 | Train Loss Unweighted: 0.3757 | Train Loss Components: Auxiliary: 0.1784 | Cardinality: 0.1000 | Classification: 0.0612 | Classification_binary: 0.1490 | HitGrouping: 0.1388 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0810 | Regression_eta: 0.0012 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0442 | Regression_eta: 0.0504 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0554 | LR: 1.06e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0334s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8932 | max_loss: 0.1579 | max_backward: 2.0333 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6262.8 | uss_MB: 5757.1 | child_rss_MB: 4183.3 | allocated_MB: 497.9 | reserved_MB: 28090.0 | inactive_split_MB: 1230.6 +rank7 [2026-10-30 06:21:38,448] INFO : Step 47400/50000 rank7 | Train Loss: 0.7705 | Train Loss Unweighted: 0.3627 | Train Loss Components: Auxiliary: 0.1629 | Cardinality: 0.0865 | Classification: 0.0634 | Classification_binary: 0.1124 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0921 | Regression_eta: 0.0010 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1077 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0600 | Regression_eta: 0.0341 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0418 | LR: 1.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4527s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.8618 | max_loss: 0.1534 | max_backward: 1.5052 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.9192 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6261.8 | uss_MB: 5756.5 | child_rss_MB: 4160.2 | allocated_MB: 494.8 | reserved_MB: 28090.0 | inactive_split_MB: 1022.5 +rank7 [2026-10-30 06:26:15,801] INFO : Step 47500/50000 rank7 | Train Loss: 0.7156 | Train Loss Unweighted: 0.3477 | Train Loss Components: Auxiliary: 0.1538 | Cardinality: 0.0805 | Classification: 0.0604 | Classification_binary: 0.1033 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0908 | Regression_eta: 0.0009 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0295 | Regression_energy: 0.0476 | Regression_eta: 0.0196 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0302 | LR: 1.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6766s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7567 | max_loss: 0.2112 | max_backward: 1.9376 | max_optimizer_step: 0.3084 | max_tensorboard_logging: 0.7886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6260.2 | uss_MB: 5758.3 | child_rss_MB: 4141.1 | allocated_MB: 496.2 | reserved_MB: 28090.0 | inactive_split_MB: 1139.8 +rank7 [2026-10-30 06:30:50,120] INFO : Step 47600/50000 rank7 | Train Loss: 0.8112 | Train Loss Unweighted: 0.3668 | Train Loss Components: Auxiliary: 0.1789 | Cardinality: 0.0951 | Classification: 0.0616 | Classification_binary: 0.1427 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0795 | Regression_eta: 0.0013 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0437 | Regression_eta: 0.0369 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0486 | LR: 1.05e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9495s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.8448 | max_loss: 0.1583 | max_backward: 2.0680 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8876 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6262.4 | uss_MB: 5759.7 | child_rss_MB: 4147.4 | allocated_MB: 498.3 | reserved_MB: 28090.0 | inactive_split_MB: 1319.2 +rank7 [2026-10-30 06:35:25,579] INFO : Step 47700/50000 rank7 | Train Loss: 0.8480 | Train Loss Unweighted: 0.3688 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.1262 | Classification: 0.0619 | Classification_binary: 0.1353 | HitGrouping: 0.1426 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0844 | Regression_eta: 0.0012 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0246 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0426 | Regression_eta: 0.0265 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0406 | LR: 1.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9103s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0755 | max_forward: 0.8768 | max_loss: 0.1499 | max_backward: 2.0237 | max_optimizer_step: 0.2103 | max_tensorboard_logging: 0.8602 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 6256.9 | uss_MB: 5757.5 | child_rss_MB: 4142.1 | allocated_MB: 502.1 | reserved_MB: 28090.0 | inactive_split_MB: 1081.8 +rank7 [2026-10-30 06:40:00,409] INFO : Step 47800/50000 rank7 | Train Loss: 0.7072 | Train Loss Unweighted: 0.3187 | Train Loss Components: Auxiliary: 0.1452 | Cardinality: 0.0751 | Classification: 0.0520 | Classification_binary: 0.1069 | HitGrouping: 0.1418 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0779 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0237 | Classification_binary: 0.0985 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0352 | Regression_eta: 0.0244 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0388 | LR: 1.04e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7811s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8594 | max_loss: 0.1546 | max_backward: 1.9999 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7690 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1394 | Memory: rss_MB: 6258.0 | uss_MB: 5759.2 | child_rss_MB: 4122.8 | allocated_MB: 494.7 | reserved_MB: 28090.0 | inactive_split_MB: 1199.6 +rank7 [2026-10-30 06:44:36,538] INFO : Step 47900/50000 rank7 | Train Loss: 0.8146 | Train Loss Unweighted: 0.3686 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.1044 | Classification: 0.0643 | Classification_binary: 0.1320 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0848 | Regression_eta: 0.0011 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0481 | Regression_eta: 0.0317 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0582 | LR: 1.04e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8502s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.8020 | max_loss: 0.1474 | max_backward: 2.0548 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.8744 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6253.1 | uss_MB: 5756.7 | child_rss_MB: 4141.1 | allocated_MB: 498.2 | reserved_MB: 28090.0 | inactive_split_MB: 1238.1 +rank7 [2026-10-30 06:49:09,636] INFO : Step 48000/50000 rank7 | Train Loss: 0.7937 | Train Loss Unweighted: 0.3728 | Train Loss Components: Auxiliary: 0.1769 | Cardinality: 0.0899 | Classification: 0.0641 | Classification_binary: 0.1440 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0811 | Regression_eta: 0.0010 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0512 | Regression_eta: 0.0197 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0381 | LR: 1.04e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8815s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0972 | max_forward: 0.8834 | max_loss: 0.1495 | max_backward: 1.9198 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.8792 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6248.5 | uss_MB: 5757.2 | child_rss_MB: 4131.1 | allocated_MB: 500.1 | reserved_MB: 30142.0 | inactive_split_MB: 1241.1 +rank7 [2026-10-30 06:49:10,727] INFO : Running validation on rank7 +rank7 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 06:49:10,727] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 06:50:01,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:50:01,896] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 06:50:27,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 06:50:28,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 06:50:29,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 06:50:30,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 06:50:40,150] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:50:41,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:50:41,779] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 06:50:57,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 06:50:58,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 06:50:59,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 06:51:00,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 06:51:10,851] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:51:10,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:51:11,430] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 06:51:27,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 06:51:27,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 06:51:28,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 06:51:29,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_48000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 06:51:39,512] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 06:56:35,097] INFO : Step 48100/50000 rank7 | Train Loss: 0.7059 | Train Loss Unweighted: 0.3376 | Train Loss Components: Auxiliary: 0.1518 | Cardinality: 0.0744 | Classification: 0.0572 | Classification_binary: 0.1139 | HitGrouping: 0.1370 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0823 | Regression_eta: 0.0010 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0334 | Regression_energy: 0.0567 | Regression_eta: 0.0221 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0294 | LR: 1.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6272s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7880 | max_loss: 0.1438 | max_backward: 1.7680 | max_optimizer_step: 0.2962 | max_tensorboard_logging: 0.7606 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 8079.1 | uss_MB: 7584.9 | child_rss_MB: 4150.7 | allocated_MB: 495.6 | reserved_MB: 27136.0 | inactive_split_MB: 1217.5 +rank7 [2026-10-30 07:01:16,421] INFO : Step 48200/50000 rank7 | Train Loss: 0.6946 | Train Loss Unweighted: 0.3232 | Train Loss Components: Auxiliary: 0.1463 | Cardinality: 0.0728 | Classification: 0.0617 | Classification_binary: 0.1066 | HitGrouping: 0.1331 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0759 | Regression_eta: 0.0010 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0415 | Regression_eta: 0.0452 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0624 | LR: 1.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3149s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.9222 | max_loss: 0.1396 | max_backward: 1.9844 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 1.1395 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1415 | Memory: rss_MB: 6287.9 | uss_MB: 5799.0 | child_rss_MB: 4134.8 | allocated_MB: 499.8 | reserved_MB: 27142.0 | inactive_split_MB: 1078.2 +rank7 [2026-10-30 07:05:53,465] INFO : Step 48300/50000 rank7 | Train Loss: 0.8330 | Train Loss Unweighted: 0.3861 | Train Loss Components: Auxiliary: 0.1830 | Cardinality: 0.1090 | Classification: 0.0673 | Classification_binary: 0.1434 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0861 | Regression_eta: 0.0011 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0471 | Regression_eta: 0.0271 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0565 | LR: 1.03e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1185s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.8487 | max_loss: 0.1562 | max_backward: 2.2195 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6294.6 | uss_MB: 5809.1 | child_rss_MB: 4124.7 | allocated_MB: 501.6 | reserved_MB: 29122.0 | inactive_split_MB: 1382.7 +rank7 [2026-10-30 07:10:30,726] INFO : Step 48400/50000 rank7 | Train Loss: 0.7675 | Train Loss Unweighted: 0.3517 | Train Loss Components: Auxiliary: 0.1657 | Cardinality: 0.0871 | Classification: 0.0600 | Classification_binary: 0.1274 | HitGrouping: 0.1401 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0806 | Regression_eta: 0.0011 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0234 | Classification_binary: 0.1089 | Regression_cos_phi: 0.0279 | Regression_energy: 0.0343 | Regression_eta: 0.0259 | Regression_pt: 0.0350 | Regression_sin_phi: 0.0259 | LR: 1.02e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8681s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7881 | max_loss: 0.2517 | max_backward: 1.9473 | max_optimizer_step: 0.3245 | max_tensorboard_logging: 0.8157 | max_tensorboard_flush: 0.0073 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6290.5 | uss_MB: 5805.3 | child_rss_MB: 4132.3 | allocated_MB: 499.3 | reserved_MB: 31138.0 | inactive_split_MB: 1204.8 +rank7 [2026-10-30 07:15:11,423] INFO : Step 48500/50000 rank7 | Train Loss: 0.7547 | Train Loss Unweighted: 0.3420 | Train Loss Components: Auxiliary: 0.1596 | Cardinality: 0.0940 | Classification: 0.0582 | Classification_binary: 0.1134 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0840 | Regression_eta: 0.0012 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0181 | Classification_binary: 0.1048 | Regression_cos_phi: 0.0258 | Regression_energy: 0.0439 | Regression_eta: 0.0233 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0203 | LR: 1.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.9618s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8499 | max_loss: 0.1446 | max_backward: 2.0748 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 1.9274 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6296.2 | uss_MB: 5809.1 | child_rss_MB: 4124.5 | allocated_MB: 498.1 | reserved_MB: 31138.0 | inactive_split_MB: 1228.5 +rank7 [2026-10-30 07:19:52,537] INFO : Step 48600/50000 rank7 | Train Loss: 0.7963 | Train Loss Unweighted: 0.3615 | Train Loss Components: Auxiliary: 0.1752 | Cardinality: 0.0922 | Classification: 0.0684 | Classification_binary: 0.1319 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0792 | Regression_eta: 0.0012 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0394 | Regression_eta: 0.0370 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0692 | LR: 1.02e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8073s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8776 | max_loss: 0.1491 | max_backward: 2.0331 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.8142 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6290.8 | uss_MB: 5807.1 | child_rss_MB: 4118.6 | allocated_MB: 495.5 | reserved_MB: 31138.0 | inactive_split_MB: 1206.2 +rank7 [2026-10-30 07:24:30,058] INFO : Step 48700/50000 rank7 | Train Loss: 0.8144 | Train Loss Unweighted: 0.3747 | Train Loss Components: Auxiliary: 0.1808 | Cardinality: 0.1052 | Classification: 0.0651 | Classification_binary: 0.1437 | HitGrouping: 0.1416 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0816 | Regression_eta: 0.0009 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0429 | Regression_eta: 0.0199 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0423 | LR: 1.02e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.9988s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0062 | max_forward: 0.9268 | max_loss: 0.2605 | max_backward: 1.8713 | max_optimizer_step: 0.3127 | max_tensorboard_logging: 0.8786 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6284.8 | uss_MB: 5803.4 | child_rss_MB: 4100.4 | allocated_MB: 497.8 | reserved_MB: 31138.0 | inactive_split_MB: 1074.2 +rank7 [2026-10-30 07:29:10,217] INFO : Step 48800/50000 rank7 | Train Loss: 0.7298 | Train Loss Unweighted: 0.3327 | Train Loss Components: Auxiliary: 0.1553 | Cardinality: 0.0895 | Classification: 0.0531 | Classification_binary: 0.1059 | HitGrouping: 0.1463 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0857 | Regression_eta: 0.0010 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0292 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0479 | Regression_eta: 0.0329 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0404 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9212s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0071 | max_forward: 0.8289 | max_loss: 0.2467 | max_backward: 1.9321 | max_optimizer_step: 0.3167 | max_tensorboard_logging: 0.8663 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6286.3 | uss_MB: 5807.4 | child_rss_MB: 4081.8 | allocated_MB: 493.2 | reserved_MB: 33190.0 | inactive_split_MB: 1200.8 +rank7 [2026-10-30 07:33:48,297] INFO : Step 48900/50000 rank7 | Train Loss: 0.7858 | Train Loss Unweighted: 0.3542 | Train Loss Components: Auxiliary: 0.1694 | Cardinality: 0.0990 | Classification: 0.0635 | Classification_binary: 0.1248 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0810 | Regression_eta: 0.0011 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0267 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0367 | Regression_eta: 0.0238 | Regression_pt: 0.0390 | Regression_sin_phi: 0.0513 | LR: 1.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7901s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0723 | max_forward: 0.8504 | max_loss: 0.1462 | max_backward: 1.8359 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.9597 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6283.9 | uss_MB: 5805.6 | child_rss_MB: 4076.8 | allocated_MB: 499.6 | reserved_MB: 33190.0 | inactive_split_MB: 1324.4 +rank7 [2026-10-30 07:38:22,688] INFO : Step 49000/50000 rank7 | Train Loss: 0.8118 | Train Loss Unweighted: 0.3751 | Train Loss Components: Auxiliary: 0.1816 | Cardinality: 0.1008 | Classification: 0.0605 | Classification_binary: 0.1456 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0834 | Regression_eta: 0.0011 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0445 | Regression_eta: 0.0453 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0403 | LR: 1.01e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8893s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0212 | max_forward: 0.8259 | max_loss: 0.1523 | max_backward: 2.0137 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.9043 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6285.9 | uss_MB: 5808.4 | child_rss_MB: 4072.7 | allocated_MB: 496.7 | reserved_MB: 33190.0 | inactive_split_MB: 1217.0 +rank7 [2026-10-30 07:38:23,699] INFO : Running validation on rank7 +rank7 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 07:38:23,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 07:39:14,059] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:39:14,628] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 07:39:39,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 07:39:40,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 07:39:41,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 07:39:42,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 07:39:52,584] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 07:39:53,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:39:53,779] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 07:40:09,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 07:40:10,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 07:40:11,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 07:40:12,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 07:40:22,539] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 07:40:23,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:40:24,012] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 07:40:39,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 07:40:40,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 07:40:40,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 07:40:41,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_49000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 07:40:51,820] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 07:45:46,210] INFO : Step 49100/50000 rank7 | Train Loss: 0.7779 | Train Loss Unweighted: 0.3682 | Train Loss Components: Auxiliary: 0.1666 | Cardinality: 0.0956 | Classification: 0.0581 | Classification_binary: 0.1132 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0973 | Regression_eta: 0.0011 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1145 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0417 | Regression_eta: 0.0269 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0403 | LR: 1.01e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6553s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.7871 | max_loss: 0.1513 | max_backward: 1.7028 | max_optimizer_step: 0.3202 | max_tensorboard_logging: 0.8112 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8221.3 | uss_MB: 7736.3 | child_rss_MB: 4079.2 | allocated_MB: 496.0 | reserved_MB: 25882.0 | inactive_split_MB: 1034.8 +rank7 [2026-10-30 07:50:30,114] INFO : Step 49200/50000 rank7 | Train Loss: 0.6904 | Train Loss Unweighted: 0.3321 | Train Loss Components: Auxiliary: 0.1478 | Cardinality: 0.0769 | Classification: 0.0586 | Classification_binary: 0.1030 | HitGrouping: 0.1404 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0843 | Regression_eta: 0.0009 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0303 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0532 | Regression_eta: 0.0215 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0299 | LR: 1.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5144s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0058 | max_forward: 0.7758 | max_loss: 0.2281 | max_backward: 1.6294 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0623 | Memory: rss_MB: 8221.3 | uss_MB: 7734.3 | child_rss_MB: 4077.6 | allocated_MB: 494.3 | reserved_MB: 27046.0 | inactive_split_MB: 1032.8 +rank7 [2026-10-30 07:55:07,372] INFO : Step 49300/50000 rank7 | Train Loss: 0.7923 | Train Loss Unweighted: 0.3629 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0976 | Classification: 0.0607 | Classification_binary: 0.1345 | HitGrouping: 0.1355 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0822 | Regression_eta: 0.0012 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0339 | Classification_binary: 0.1188 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0324 | Regression_eta: 0.0216 | Regression_pt: 0.0336 | Regression_sin_phi: 0.0287 | LR: 1.00e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8692s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0947 | max_forward: 0.8714 | max_loss: 0.1494 | max_backward: 1.8934 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6372.1 | uss_MB: 5898.0 | child_rss_MB: 4046.3 | allocated_MB: 496.4 | reserved_MB: 27936.0 | inactive_split_MB: 1032.5 +rank7 [2026-10-30 07:59:43,350] INFO : Step 49400/50000 rank7 | Train Loss: 0.7767 | Train Loss Unweighted: 0.3618 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0910 | Classification: 0.0609 | Classification_binary: 0.1269 | HitGrouping: 0.1402 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0011 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0432 | Regression_eta: 0.0282 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0361 | LR: 1.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8897s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7921 | max_loss: 0.1522 | max_backward: 1.8733 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 1.0530 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6372.1 | uss_MB: 5897.4 | child_rss_MB: 4046.2 | allocated_MB: 499.3 | reserved_MB: 27942.0 | inactive_split_MB: 945.0 +rank7 [2026-10-30 08:04:24,758] INFO : Step 49500/50000 rank7 | Train Loss: 0.6936 | Train Loss Unweighted: 0.3266 | Train Loss Components: Auxiliary: 0.1503 | Cardinality: 0.0886 | Classification: 0.0575 | Classification_binary: 0.1025 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0821 | Regression_eta: 0.0008 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0169 | Classification_binary: 0.0954 | Regression_cos_phi: 0.0225 | Regression_energy: 0.0420 | Regression_eta: 0.0179 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0242 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3346s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7758 | max_loss: 0.1430 | max_backward: 1.5171 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.9052 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6349.0 | uss_MB: 5893.9 | child_rss_MB: 3988.7 | allocated_MB: 494.3 | reserved_MB: 27942.0 | inactive_split_MB: 993.1 +rank7 [2026-10-30 08:08:58,923] INFO : Step 49600/50000 rank7 | Train Loss: 0.7780 | Train Loss Unweighted: 0.3483 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0981 | Classification: 0.0640 | Classification_binary: 0.1357 | HitGrouping: 0.1360 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0732 | Regression_eta: 0.0010 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0354 | Regression_eta: 0.0248 | Regression_pt: 0.0365 | Regression_sin_phi: 0.0395 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8756s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.1132 | max_forward: 0.9144 | max_loss: 0.1603 | max_backward: 1.9243 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8566 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6337.0 | uss_MB: 5899.6 | child_rss_MB: 3955.3 | allocated_MB: 498.5 | reserved_MB: 27942.0 | inactive_split_MB: 1179.8 +rank7 [2026-10-30 08:13:36,932] INFO : Step 49700/50000 rank7 | Train Loss: 0.7707 | Train Loss Unweighted: 0.3541 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0965 | Classification: 0.0650 | Classification_binary: 0.1281 | HitGrouping: 0.1366 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0793 | Regression_eta: 0.0010 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0447 | Regression_eta: 0.0365 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0465 | LR: 1.00e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7052s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.8277 | max_loss: 0.2489 | max_backward: 2.1335 | max_optimizer_step: 0.2202 | max_tensorboard_logging: 0.5911 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6335.5 | uss_MB: 5898.2 | child_rss_MB: 3952.0 | allocated_MB: 499.4 | reserved_MB: 27942.0 | inactive_split_MB: 1189.8 +rank7 [2026-10-30 08:18:11,416] INFO : Step 49800/50000 rank7 | Train Loss: 0.6962 | Train Loss Unweighted: 0.3173 | Train Loss Components: Auxiliary: 0.1441 | Cardinality: 0.0781 | Classification: 0.0546 | Classification_binary: 0.1045 | HitGrouping: 0.1413 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0773 | Regression_eta: 0.0011 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0302 | Classification_binary: 0.0883 | Regression_cos_phi: 0.0236 | Regression_energy: 0.0421 | Regression_eta: 0.0258 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0244 | LR: 1.00e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5283s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7641 | max_loss: 0.1456 | max_backward: 1.8519 | max_optimizer_step: 0.2140 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6335.5 | uss_MB: 5895.8 | child_rss_MB: 3955.9 | allocated_MB: 493.2 | reserved_MB: 27942.0 | inactive_split_MB: 1036.8 +rank7 [2026-10-30 08:22:53,416] INFO : Step 49900/50000 rank7 | Train Loss: 0.8005 | Train Loss Unweighted: 0.3732 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0648 | Classification_binary: 0.1307 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0010 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0294 | Regression_energy: 0.0542 | Regression_eta: 0.0249 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0309 | LR: 1.00e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.2008s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0080 | max_forward: 0.8025 | max_loss: 0.6746 | max_backward: 2.4070 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.8891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6336.4 | uss_MB: 5896.7 | child_rss_MB: 3952.1 | allocated_MB: 498.5 | reserved_MB: 29062.0 | inactive_split_MB: 960.8 +rank7 [2026-10-30 08:27:37,250] INFO : Step 50000/50000 rank7 | Train Loss: 0.8463 | Train Loss Unweighted: 0.3713 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1316 | Classification: 0.0570 | Classification_binary: 0.1506 | HitGrouping: 0.1393 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0802 | Regression_eta: 0.0010 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5901 | Classification_binary: 1.0000 | Regression_cos_phi: 75.6069 | Regression_energy: 0.5238 | Regression_eta: 28.3978 | Regression_pt: 0.5374 | Regression_sin_phi: 73.5171 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0442 | Regression_eta: 0.0236 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0351 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3470s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8673 | max_loss: 0.6600 | max_backward: 2.5297 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.7763 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6334.4 | uss_MB: 5899.5 | child_rss_MB: 3934.3 | allocated_MB: 499.0 | reserved_MB: 29062.0 | inactive_split_MB: 949.0 +rank7 [2026-10-30 08:27:38,187] INFO : Running validation on rank7 +rank7 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 08:27:38,187] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 08:28:29,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 08:28:29,585] INFO : Running predictions on clic_edm_ttbar_hits +rank7 [2026-10-30 08:29:00,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 08:29:01,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 08:29:02,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 08:29:03,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 08:29:13,153] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 08:29:13,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 08:29:14,403] INFO : Running predictions on clic_edm_ww_fullhad_hits +rank7 [2026-10-30 08:29:30,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 08:29:32,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 08:29:33,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 08:29:35,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 08:29:45,134] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-30 08:29:45,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 08:29:46,216] INFO : Running predictions on clic_edm_qq_hits +rank7 [2026-10-30 08:30:01,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 08:30:02,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 08:30:03,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 08:30:04,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_hits_heptv2_seed12345_20261028_111202_012645/preds_step_50000/clic_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 08:30:14,436] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 08:30:15,174] INFO : Training completed. Total time on device 7: 1059.927min diff --git a/clic/hits/v3.2.0/comparison_metrics/provenance.json b/clic/hits/v3.2.0/comparison_metrics/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd93dc97277643919f983f16a0633812762afa4 --- /dev/null +++ b/clic/hits/v3.2.0/comparison_metrics/provenance.json @@ -0,0 +1,24 @@ +{ + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "study": "20261030_clic_cld_checkpoint_summary", + "selected_step": 50000, + "sources": "Original per-run history JSON and train TensorBoard streams, retained alongside these summaries.", + "metrics": { + "f1": "Count-aggregated validation particle F1. Capped matching cost 2.0, acceptance deltaR<0.1 and relative pT error<0.5, no PID requirement.", + "pt_mae": "Mean absolute relative pT error among accepted pairs; conditional and truncated by acceptance, not inclusive resolution.", + "jet_iqr": "Equal mean of ttbar, WW fullhad, qq IQRs of angular-matched predicted/target jet pT ratios.", + "jet_recall": "Equal-process response-qualified jet recall, with abs(pT ratio - 1)<0.5.", + "met": "Full-validation mean absolute error of MET magnitude in GeV.", + "p3": "Detailed 512-entry subset mean norm of event three-momentum difference, GeV.", + "steady_s": "Median seconds/step across adjacent 100-step train TensorBoard intervals, excluding validation boundaries, warmup and session changes.", + "compute_s": "Median instrumented rank-zero forward+loss+backward+optimizer time, seconds.", + "peak_gib": "Maximum recorded allocator peak across ranks and sessions, per logical device. May include prior validation allocations." + }, + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ] +} diff --git a/clic/hits/v3.2.0/comparison_metrics/study_methodology.md b/clic/hits/v3.2.0/comparison_metrics/study_methodology.md new file mode 100644 index 0000000000000000000000000000000000000000..d40607cfea1c2d71542e91632ef20b56a9c172a0 --- /dev/null +++ b/clic/hits/v3.2.0/comparison_metrics/study_methodology.md @@ -0,0 +1,207 @@ +# CLIC and CLD checkpoint summary, 30 October 2026 + +Twenty summary slides compare the eight runs in the 28 October +`clic_cld_backbone_benchmark` campaign. The common endpoint is **50,000 optimizer +steps**. Evolution curves receive priority over endpoint-only comparisons. +This is a snapshot of local results, not a report of current remote job status. + +## Deliverables + +- [PDF slides](output/checkpoint_summary.slides.pdf) +- [Self-contained HTML slides](output/checkpoint_summary.slides.html), arrow-key navigation +- [Source notebook](checkpoint_summary.ipynb) and [executed notebook](output/checkpoint_summary.executed.ipynb) +- [Numerical scorecard](output/scorecard.csv), [summary](output/summary.json), [resource series](output/resources.json) +- `output/slides.json`: generated slide copy, figure paths and source categories +- `output/*.svg` and `output/*.png`: standalone scientific figures + +The figures are generated from archived data by `study_analysis.py`. Tables and +slide copy are HTML text. No external CDN, mutable experiment path, or earlier +study archive is needed to render. This deliverable follows the notebook/HTML/PDF +format in the parent README; it does not include a PowerPoint conversion. + +## Findings + +HEPTv2 improves final jet IQR, MET error and detailed event momentum error in +all four detector/representation pairs. On CLIC hits, F1 rises from 0.7747 to +0.8623 (+8.76 percentage points) and the equal-process jet IQR decreases from +0.1579 to 0.0995 (37.0%). On CLD hits, F1 rises from 0.8458 to 0.8624 +(+1.67 points) and jet IQR decreases from 0.1243 to 0.1040 (16.4%). + +Hit HEPTv2 has higher final particle F1 than track/cluster HEPTv2 in both +detectors. Track/cluster HEPTv2 retains lower conditional particle momentum +error, jet IQR, MET error and event momentum-vector error. These statements +refer to the recorded metrics and configured target populations, not paired +statistical significance or full inclusive momentum resolution. + +The median ordinary training-interval estimates are about 0.11 s/step for PF +attention, 0.40–0.41 s for PF HEPTv2, 2.36–2.38 s for hit attention and +2.84–2.89 s for hit HEPTv2. PF uses one logical accelerator device; hits use +eight. At 50k steps this projects to roughly 55–58 times the device cost for +hit HEPTv2 versus PF HEPTv2, excluding validation, queueing and input +preprocessing. This is not an inference or end-to-end reconstruction benchmark. +Exposed rank-zero batch wait has medians near 2 ms for PF and 17 ms for hits. +No storage byte counters, CPU utilization or bandwidth measurements support a +broader I/O-bottleneck conclusion. + +## Runs and configuration + +All runs use seed 12345, revision +`6b80b983477321cf708e5043fb1d2ec15dd398bc`, global batch 128, 50k scheduled steps, +LAMB, BF16, LR 0.001 with cosine decay, and no compilation. All completed +histories end at 50k. Run IDs record launch preparation on 28 October 2026 at +11:12; hit continuations were submitted on 29 October as recorded in their +manifests. Full log timestamps and exact commands remain archived. + +| Archive name | Exact run ID | Devices | History points | Resume checkpoint | +| --- | --- | ---: | ---: | --- | +| cld_hits_attention | `cld_hits_attention_seed12345_20261028_111201_979187` | 8 | 50 | 35000 | +| cld_hits_heptv2 | `cld_hits_heptv2_seed12345_20261028_111202_875252` | 8 | 49 | 30000 | +| cld_pf_attention | `cld_pf_attention_seed12345_20261028_111201_967206` | 1 | 50 | none | +| cld_pf_heptv2 | `cld_pf_heptv2_seed12345_20261028_111202_024020` | 1 | 50 | none | +| clic_hits_attention | `clic_hits_attention_seed12345_20261028_111202_121851` | 8 | 50 | 35000 | +| clic_hits_heptv2 | `clic_hits_heptv2_seed12345_20261028_111202_012645` | 8 | 50 | 29000 | +| clic_pf_attention | `clic_pf_attention_seed12345_20261028_111202_052283` | 1 | 50 | none | +| clic_pf_heptv2 | `clic_pf_heptv2_seed12345_20261028_111202_849554` | 1 | 50 | none | + +Hardware provenance comes from `scenario-manifest.json`: platform +`lumi_mi250x`, eight logical devices available per platform/node. Actual model +world size is one for track/cluster and eight for hits, verified against resolved +configuration and logs. The generic `gpu_type: l40` in train-config is a stale +recipe default, not evidence that these jobs ran on L40 hardware. Device-hours +count logical MI250X accelerator devices/GCDs, not whole dual-GCD packages. + +All models use width 256, six shared backbone layers. PF attention/HEPTv2 have +3,114,258 / 8,639,954 parameters. Hit attention/HEPTv2 have +7,260,244 / 12,786,964. The hit decoder uses four global-Flash layers and 256 +slots, supervised grouping, tracker-neighborhood features and dedicated tracker +momentum readout. HEPTv2 uses deterministic hashing, block size 128 and activation +checkpointing. These are fixed recipe comparisons, not equal-parameter or +pure-backbone-kernel benchmarks. Losses/objectives and readouts differ between +PF and hit models, so weighted loss is not used as cross-representation evidence. + +Both detectors use ttbar, WW fullhad and qq. Dataset names are +`{clic,cld}_edm_{ttbar,ww_fullhad,qq}_{pf,hits}`, with all ten numbered configs. +PF TFDS is 3.3.0 and hit TFDS is 3.2.2. Exact production/data locations are +preserved only as provenance in original configuration files. Rendering never +reads those locations. At 50k each run has about 2.37 training epochs / 6.4M +training-event presentations, not 6.4M unique events. + +## Metrics and sampling + +Particle curves use history `valid/metrics/particle/*` scalars. The archived +validation matcher minimizes `(deltaR/0.1)^2 + (log(pT ratio)/log(2))^2`, capped +at 2 per pair. Accepted matches require deltaR < 0.1 and relative pT error < 0.5, +without PID agreement. F1/efficiency aggregate counts through the validation +accumulator, rather than averaging process-level F1. Conditional momentum MAE +is mean absolute relative pT error among accepted pairs; the 50% acceptance cut +truncates its tails. It is not a central-68% width or inclusive resolution. + +`nvalid=512` with ten split configs selects 51 entries per config, 510 per +process, 1,530 total. The distributed sampler pads the hit validation collection +to 1,536 entries. Detailed momentum metrics apply to four batches per rank, +512 entries globally for both representations, but rank partitioning changes +which events are in this subset. `momentum/all/event/p3_error_mean` is the mean +norm of the difference between summed predicted and target three-momenta, +not the mean of individual particle momentum errors. Basic MET and energy +metrics use the full validation collection. MET error compares magnitudes. + +Jet curves use top-level `step//jet_ratio/jet_ratio_target_to_pred_pt/*` +scalars from the test/plot cycle. Ratios are predicted / target pT. IQR is +Q75−Q25 among angular-matched jets, with no additional response acceptance cut. +Response-qualified recall counts angular matches satisfying |ratio−1| < 0.5 +relative to all target jets. Each displayed curve is an equal mean of the three +process-specific summaries, not an IQR of pooled jets. `ntest=512` likewise +requests 51 events from each of ten configs per process before distributed +padding; sample identity and duplicate entries were not independently audited. + +PF and hit target means differ slightly (e.g. CLD 70.565 versus 70.655), which +can include both target-definition and sampler effects. We have not verified +exact truth-event identity across representations, nor rescored predictions +with a common event subset. Backbone comparisons within a representation have +the same nominal configuration and validation counts, but have no paired-event +confidence intervals or training-seed replication. Repeated validation/test +inspection makes the comparisons exploratory. Earlier studies have different +data and matching defaults, and their metric values are not mixed into this +campaign. CLD hit HEPTv2 lacks `step_30000.json`; plots connect 29k to 31k without +synthesizing a 30k observation. + +## Resource method + +`extract_resources.py` reads every archived train event file separately with +unbounded scalar retention. It retains the latest wall-time observation for +replayed steps, preserving stream identity to reject cross-session intervals. +All required scalar series have aligned step sets. There are 500 observations +per model at steps 100–50,000 after deduplication. + +Ordinary throughput uses elapsed wall time between adjacent 100-step TensorBoard +observations from the same event file, excluding intervals beginning at a +multiple of 1,000 (which cross validation/test work), intervals before 1,000, +and nonadjacent/restart boundaries. The median seconds per optimizer step is +robust to isolated stalls but is an estimate, not total campaign elapsed time. +Each 100-step interval still contains one diagnostic step. Projected 50k device +cost multiplies this median by 50,000 and by actual world size. It excludes +validation, test plotting, checkpoint cycles at excluded boundaries, scheduling +downtime, startup and abandoned/replayed training work. It does not include PF +tracking/clustering preprocessing. + +Instrumented-step curves and phase breakdowns use rank-zero raw phase timers, +not sums of maxima across different ranks. Compute is forward + loss + backward ++ optimizer. The enclosing `time_model_forward` includes training, backward and +logging, so it is not used as pure forward. `step/time_train_total` includes +synchronized diagnostics and memory logging. Its median can be much greater +than the ordinary interval estimate. The remaining phase is total minus listed +phases; negative residuals are rejected. It includes diagnostic and collective +work not separately plotted. + +Loader wait is the exposed wall time in `next(train_iterator)` on rank zero. +Transfer is the synchronized batch-to-device phase. Neither measures storage +service time, byte throughput, asynchronous worker effort or the slowest rank's +I/O. Resource plots use all raw sampled observations; thick timing lines are +centered 11-sample medians. Input multiplicities quoted on slide 15 are medians +of the per-step, across-rank batch-average element counts. + +Allocator memory is the maximum across ranks of historical peak allocation at +step end, and can include earlier validation allocations. Restart resets it. +This differs from reserved/cached memory and total VRAM occupancy. CPU RSS is +the mean training-process footprint across ranks; it omits child RSS and is not +unique node memory. MB tag values use MiB, divided by 1024 for GiB. The compact +resource JSON also retains child RSS, USS, current reserved memory and padding +for later inspection. No inference benchmark is inferred from these logs. + +**Timer trap:** history `step_valid_time` is calculated as current time minus +session start minus the last training-step duration. It is cumulative and +resets on resume, not a validation-cycle duration. Neither it nor a sum of +history `step_total_time` values is used as validation or total training cost. +The archived revision's `training.py` documents this behavior. + +## Archive and reproduction + +Run from the repository root: + +```bash +bash notebooks/studies/20261030_clic_cld_checkpoint_summary/render.sh +``` + +The script executes the source notebook, which re-extracts resources, regenerates +figures/tables/copy, and emits each slide. It writes the executed notebook and +self-contained HTML, then headless Chrome generates PDF and Poppler renders page +PNGs. Override `PYTHON_BIN` / `CHROME_BIN` if needed. Dependencies are nbformat, +nbclient, IPython, numpy, matplotlib, TensorBoard and a Python 3 Jupyter kernel. +Jupyter needs local socket permissions. Chrome/Chromium is optional for HTML +but required for PDF. `MPLCONFIGDIR` defaults to a temporary writable cache; +scientific inputs and outputs remain study-relative. + +`inputs//` contains complete histories, complete +`plots_step_50000`, the rank-zero `train.log.0` copied as `train.log`, separate +nonempty `tensorboard/train` and `tensorboard/valid` streams, train configuration, +hyperparameters, scenario manifest and resolved particleflow specification. +Files are byte-for-byte copies. `inputs/run_manifest.json` maps descriptive names +to exact original run IDs. `inputs/SHA256SUMS` checks every archived input. +`inputs/sources/` contains the campaign revision's metric, data-sampling and +timing code for definition auditing. No checkpoint, prediction parquet, model +pickle or dataset cache was copied. Inputs total approximately 428.6 MB, mainly +TensorBoard. There is no dependency on a previous study's local artifacts. + +New study files follow the parent directory's local-artifact/Git-ignore policy. +Preserve the full dated folder to retain reproducibility. No private-bucket +upload was requested or performed. diff --git a/clic/hits/v3.2.0/comparison_metrics/summary.json b/clic/hits/v3.2.0/comparison_metrics/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4f001ffb8f5c3c80b2a813b2db9210916c5ba5fa --- /dev/null +++ b/clic/hits/v3.2.0/comparison_metrics/summary.json @@ -0,0 +1,44 @@ +[ + { + "run": "clic_hits_attention", + "detector": "CLIC", + "model": "Hits attention", + "f1": 0.7747290134429932, + "efficiency": 0.7542423009872437, + "pt_mae": 0.15970812737941742, + "jet_iqr": 0.15794942728739483, + "jet_recall": 0.9233278633357459, + "met": 7.830078125, + "p3": 21.845767974853516, + "steady_s": 2.355925884246826, + "devices": 8, + "params": 7260244, + "wait_ms": 17.113685607910156, + "wait_pct": 0.35407143366718336, + "transfer_ms": 5.6031260173767805, + "compute_s": 2.3847950026392937, + "sampled_s": 4.924937963485718, + "peak_gib": 11.855772018432617 + }, + { + "run": "clic_hits_heptv2", + "detector": "CLIC", + "model": "Hits HEPTv2", + "f1": 0.8623151779174805, + "efficiency": 0.8534561395645142, + "pt_mae": 0.1199609786272049, + "jet_iqr": 0.09948254751602414, + "jet_recall": 0.9528668406660036, + "met": 4.94622278213501, + "p3": 14.568695068359375, + "steady_s": 2.88767199754715, + "devices": 8, + "params": 12786964, + "wait_ms": 16.89016819000244, + "wait_pct": 0.31290290098008755, + "transfer_ms": 5.54687948897481, + "compute_s": 2.8723186030983925, + "sampled_s": 5.453466176986694, + "peak_gib": 14.498655319213867 + } +] diff --git a/clic/hits/v3.2.0/release.json b/clic/hits/v3.2.0/release.json new file mode 100644 index 0000000000000000000000000000000000000000..748e07cc0e16b29c9a3f28055601195fdad00abc --- /dev/null +++ b/clic/hits/v3.2.0/release.json @@ -0,0 +1,72 @@ +{ + "release": "3.2.0", + "detector": "clic", + "representation": "hits", + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "selected_step": 50000, + "dataset_version": "3.2.2", + "models": { + "attention": { + "run": "clic_hits_attention_seed12345_20261028_111202_121851", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "75f3976dcdcc42c0bc91a5dc418775e4d166ae08f1f19cb8e7ce2846e6302f57", + "parameters": 7260244, + "global_batch_size": 128, + "accelerator_devices": 8, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.7747290134429932, + "metrics/particle/matched/pt_relative_abs_mean": 0.15970812737941742, + "metrics/particle/event/met_abs_error": 7.830078125, + "metrics/particle/momentum/all/event/p3_error_mean": 21.845767974853516 + } + }, + "heptv2": { + "run": "clic_hits_heptv2_seed12345_20261028_111202_012645", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "00392bfbe6a32174783b909254db2ed09959cb988650c35d48fd29d51e5dbf75", + "parameters": 12786964, + "global_batch_size": 128, + "accelerator_devices": 8, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8623151779174805, + "metrics/particle/matched/pt_relative_abs_mean": 0.1199609786272049, + "metrics/particle/event/met_abs_error": 4.94622278213501, + "metrics/particle/momentum/all/event/p3_error_mean": 14.568695068359375 + } + } + }, + "contents": "Final 50k checkpoints, complete available histories, separate train/valid TensorBoard streams, configs, model kwargs, scenario manifests, all training logs, and complete final plots. Prediction parquet files and batch dumps are excluded.", + "checkpoint_selection": "Final scheduled 50k checkpoint; not selected as best by validation or test performance.", + "code_tag": "v3.2.0 is the intended code release label. This upload does not create or verify the Git tag. The training commit above is authoritative.", + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ], + "checkpoint_validation": [ + { + "run": "clic_hits_attention_seed12345_20261028_111202_121851", + "step": 50000, + "parameters": 7260244, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + }, + { + "run": "clic_hits_heptv2_seed12345_20261028_111202_012645", + "step": 50000, + "parameters": 12786964, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + } + ] +}